Regenerated dictionary and started showing guesses

cmd/prep-words/main.go

@@ -5,24 +5,31 @@ import (
 	"bytes"
 	"encoding/json"
 	"flag"
+	"github.com/PuerkitoBio/goquery"
+	gonanoid "github.com/matoous/go-nanoid"
 	"log"
 	"math/rand/v2"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
+	"strings"
 	"time"
 )
 
+var validWordRegex = regexp.MustCompile(`^[a-z]+$`)
+
 type wordList struct {
 	Words map[int][]string `json:"words"`
 }
 
 type shufflePattern struct {
+	ID    string        `json:"id"`
 	Index map[int][]int `json:"index"`
 }
 
 func main() {
-	dictFile := flag.String("dict", "./dict/en_GB.dic", "dictionary of word to prep")
+	dictFile := flag.String("dict", "./dict", "directory of dictionary of word to prep")
 	outDir := flag.String("out", "./site/assets/data", "output directory")
 	flag.Parse()
 
@@ -32,14 +39,24 @@ func main() {
 		Words: make(map[int][]string),
 	}
 
+	wordSet := make(map[string]bool)
 	if err := scanSuitableWords(*dictFile, func(word string) {
-		if len(word) >= 4 && len(word) <= 6 {
-			words.Words[len(word)] = append(words.Words[len(word)], word)
+		w := strings.TrimSpace(word)
+		if !validWordRegex.MatchString(w) {
+			return
+		}
+
+		if len(w) >= 4 && len(w) <= 6 {
+			wordSet[word] = true
 		}
 	}); err != nil {
 		log.Fatal(err)
 	}
 
+	for w := range wordSet {
+		words.Words[len(w)] = append(words.Words[len(w)], w)
+	}
+
 	for k, word := range words.Words {
 		log.Printf("Found %d words of length %v", len(word), k)
 		sort.Strings(word)
@@ -54,7 +71,10 @@ func main() {
 	}
 
 	// Generate a shuffle pattern
-	shp := shufflePattern{Index: make(map[int][]int)}
+	shp := shufflePattern{
+		ID:    gonanoid.MustID(12),
+		Index: make(map[int][]int),
+	}
 	for k := range words.Words {
 		pattern := make([]int, len(words.Words[k]))
 		for i := range words.Words[k] {
@@ -80,7 +100,56 @@ func main() {
 	}
 }
 
-func scanSuitableWords(dictFile string, withWord func(word string)) error {
+func scanSuitableWords(dictDir string, withWord func(word string)) error {
+	if err := scanSuitableWordsFromWordListHTML(filepath.Join(dictDir, "oxford-word-list.htm"), withWord); err != nil {
+		return err
+	}
+
+	if err := scanSuitableWordsFromOxford3000(filepath.Join(dictDir, "The_Oxford_3000.txt"), withWord); err != nil {
+		return err
+	}
+
+	if err := scanSuitableWordsFromEnGB(filepath.Join(dictDir, "en_GB.dic"), withWord); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func scanSuitableWordsFromWordListHTML(dictFile string, withWord func(word string)) error {
+	f, err := os.Open(dictFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	dom, err := goquery.NewDocumentFromReader(f)
+	if err != nil {
+		return err
+	}
+
+	dom.Find("table.t tbody tr td.t:nth-child(2)").Each(func(i int, s *goquery.Selection) {
+		withWord(s.Text())
+	})
+	return nil
+}
+
+func scanSuitableWordsFromOxford3000(dictFile string, withWord func(word string)) error {
+	f, err := os.Open(dictFile)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		withWord(scanner.Text())
+	}
+
+	return scanner.Err()
+}
+
+func scanSuitableWordsFromEnGB(dictFile string, withWord func(word string)) error {
 	f, err := os.Open(dictFile)
 	if err != nil {
 		return err