/playlist/encoding.go

package playlist

import (
	"unicode"
	"unicode/utf8"

	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
)

// legacyEncodings lists codepages commonly used by music taggers that wrote
// non-Latin text but marked the encoding as Latin-1.
var legacyEncodings = []encoding.Encoding{
	charmap.Windows1255, // Hebrew
	charmap.Windows1256, // Arabic
	charmap.Windows1251, // Cyrillic
	charmap.Windows1253, // Greek
	charmap.Windows874,  // Thai
}

// sanitizeTag detects mojibake from legacy codepages and re-decodes to UTF-8.
//
// Many old ID3 taggers write non-Latin text using a Windows codepage but mark
// the encoding as Latin-1. The tag library faithfully decodes those bytes as
// Latin-1, producing garbled text. We detect this by checking for a high
// density of Latin-1 supplement characters (U+0080–U+00FF), reverse the
// Latin-1 decode to recover the original bytes, then try common codepages
// and pick the one that produces the most non-Latin script characters.
func sanitizeTag(s string) string {
	if s == "" {
		return s
	}

	// Count runes in the Latin-1 supplement range (U+0080–U+00FF).
	// Real Latin text (French, German) rarely exceeds ~10% accented chars.
	// Mojibake from non-Latin scripts is almost entirely in this range.
	var total, highCount int
	for _, r := range s {
		total++
		if r >= 0x80 && r <= 0xFF {
			highCount++
		}
	}
	if total == 0 || highCount*3 < total {
		return s
	}

	// Reverse the Latin-1 decode to recover the original tag bytes.
	raw := make([]byte, 0, total)
	for _, r := range s {
		if r > 0xFF {
			return s // Contains runes outside Latin-1 — not simple mojibake
		}
		raw = append(raw, byte(r))
	}

	// The original bytes might be valid UTF-8 that was double-decoded.
	if utf8.Valid(raw) {
		return string(raw)
	}

	// Try legacy codepages; pick the one that produces the most
	// non-Latin letters (Hebrew, Cyrillic, Arabic, Greek, Thai, etc.).
	var bestText string
	var bestScore int
	for _, enc := range legacyEncodings {
		decoded, err := enc.NewDecoder().Bytes(raw)
		if err != nil {
			continue
		}
		text := string(decoded)
		if !utf8.ValidString(text) {
			continue
		}
		score := 0
		for _, r := range text {
			if unicode.IsLetter(r) && r > 0x024F {
				score++
			}
		}
		if score > bestScore {
			bestScore = score
			bestText = text
		}
	}

	if bestText != "" {
		return bestText
	}
	return s
}