package calibre import ( "crypto/hmac" "crypto/md5" "embed" "encoding/hex" "encoding/json" "encoding/xml" "fmt" "io" "io/fs" "log" "math" "os" "path/filepath" "sort" "strings" "sync" // "github.com/cespare/xxhash" "github.com/karrick/godirwalk" uuid "github.com/satori/go.uuid" ) var ( //go:embed embResources embResources embed.FS ) func c(err error, m string) { if err != nil { log.Fatal(err, m) } } type HugoCatalogItem struct { Key string Book *BookJSON } type DataJs struct { Portable bool `json:"portable"` Total int `json:"total,omitempty"` Books []*BookJSON `json:"books"` } // type BooksJSON struct { // Books []*BookJSON `json:"books"` // } type IdentifierJSON struct { Scheme string `json:"scheme"` Code string `json:"code"` } type FormatJSON struct { Format string `json:"format"` DirPath string `json:"dir_path"` FileName string `json:"file_name"` Size int64 `json:"size"` XXHash uint64 `json:"xxhash,omitempty"` } type BookJSON struct { Id string `json:"_id"` Librarian string `json:"librarian"` LibraryUUID string `json:"library_uuid"` Title string `json:"title"` TitleSort string `json:"title_sort"` Authors []string `json:"authors"` Abstract string `json:"abstract"` Tags []string `json:"tags"` Publisher string `json:"publisher"` Pubdate string `json:"pubdate"` LastModified string `json:"last_modified"` Languages []string `json:"languages"` Identifiers []*IdentifierJSON `json:"identifiers"` Formats []*FormatJSON `json:"formats"` CoverUrl string `json:"cover_url"` } type Creator struct { Role string `xml:"opf:role,attr"` Name string `xml:",chardata"` } type Identifier struct { Scheme string `xml:"opf:scheme,attr"` Id string `xml:"id,attr"` // Calibre has two internal Ids: calibre_id and uuid_id Value string `xml:",chardata"` } type BookOpfW struct { XMLName xml.Name `xml:"package"` Version string `xml:"version,attr"` Xmlns string `xml:"xmlns,attr"` UniqueIdentifier string `xml:"unique-identifier,attr"` Metadata struct { DC string `xml:"xmlns:dc,attr"` OPF string `xml:"xmlns:opf,attr"` Identifiers []struct { Scheme string `xml:"opf:scheme,attr"` Id string `xml:"id,attr"` // Calibre has two internal Ids: calibre_id and uuid_id Value string `xml:",chardata"` } `xml:"dc:identifier"` Title string `xml:"dc:title"` Creators []struct { Role string `xml:"opf:role,attr"` Name string `xml:",chardata"` } `xml:"dc:creator"` Published string `xml:"dc:date"` Description string `xml:"dc:description,omitempty"` Publisher string `xml:"dc:publisher,omitempty"` Languages []struct { Language string `xml:",chardata"` } `xml:"dc:language"` Tags []struct { Tag string `xml:",chardata"` } `xml:"dc:subject"` Meta []struct { Content string `xml:"content,attr"` Name string `xml:"name,attr"` } `xml:"meta"` } `xml:"metadata"` } type BookOpf struct { XMLName xml.Name `xml:"package"` Version string `xml:"version,attr"` Xmlns string `xml:"xmlns,attr"` UniqueIdentifier string `xml:"unique-identifier,attr"` Metadata struct { DC string `xml:"dc,attr"` OPF string `xml:"opf,attr"` Identifiers []struct { Scheme string `xml:"scheme,attr"` Id string `xml:"id,attr"` // Calibre has two internal Ids: calibre_id and uuid_id Value string `xml:",chardata"` } `xml:"identifier"` Title string `xml:"title"` Creators []struct { Role string `xml:"role,attr"` Name string `xml:",chardata"` } `xml:"creator"` Published string `xml:"date"` Description string `xml:"description"` Publisher string `xml:"publisher"` Languages []struct { Language string `xml:",chardata"` } `xml:"language"` Tags []struct { Tag string `xml:",chardata"` } `xml:"subject"` Meta []struct { Content string `xml:"content,attr"` Name string `xml:"name,attr"` } `xml:"meta"` } `xml:"metadata"` } // TitleSort returns if Calibre processed Title for sorting order func (book BookOpf) TitleSort() string { for _, meta := range book.Metadata.Meta { if meta.Name == "calibre:title_sort" { return meta.Content } } return "" } // LastModified returns when Calibre last time touched the file func (book BookOpf) LastModified() string { lastModified := "" for _, meta := range book.Metadata.Meta { if meta.Name == "calibre:timestamp" { lastModified = meta.Content } } return lastModified } // Authors parses creators and give back authors func (book BookOpf) Authors() []string { authors := []string{} if len(book.Metadata.Creators) > 0 { for _, author := range book.Metadata.Creators { if author.Role == "aut" { authors = append(authors, author.Name) } } } return authors } // Tags gets a list of tags from Subject nodes in OPF func (book BookOpf) Tags() []string { tags := []string{} for _, tag := range book.Metadata.Tags { tags = append(tags, tag.Tag) } return tags } // Languages gets a list of languages func (book BookOpf) Languages() []string { languages := []string{} for _, l := range book.Metadata.Languages { languages = append(languages, l.Language) } return languages } // Identifiers returns list of identifiers like ISBN, DOI, Google, Amazon... func (book BookOpf) Identifiers() []*IdentifierJSON { identifiers := []*IdentifierJSON{} for _, i := range book.Metadata.Identifiers { if i.Id == "" { identifiers = append(identifiers, &IdentifierJSON{ Scheme: strings.ToLower(i.Scheme), Code: i.Value, }) } } return identifiers } // Uuid returns calibre's book uuid... func (book BookOpf) Uuid(librarySecret string) string { for _, i := range book.Metadata.Identifiers { if i.Id == "uuid_id" { h := hmac.New(md5.New, []byte(librarySecret)) h.Write([]byte(i.Value)) sha := hex.EncodeToString(h.Sum(nil)) u, err := uuid.FromString(sha) if err != nil { fmt.Println(err) } return u.String() } } return "" } // Formats return list of files and info on them func Formats(calibrePath, relativeDirPath string) []*FormatJSON { path := filepath.Join(calibrePath, relativeDirPath) if strings.HasSuffix(path, "/") != true { path = path + "/" } formats := []*FormatJSON{} files, err := os.ReadDir(path) c(err, "list of formats...") for _, f := range files { if f.Name() != "metadata.opf" && f.Name() != "cover.jpg" { fi, err := f.Info() c(err, "file info in Formats") x, err := os.Open(filepath.Join(path, f.Name())) c(err, "open file in Formats") defer x.Close() // xxHash := xxhash.New() // _, err = io.Copy(xxHash, x) // c(err, "io.Copy xxHash in Formats") formats = append(formats, &FormatJSON{ Format: strings.ReplaceAll(filepath.Ext(f.Name()), ".", ""), DirPath: relativeDirPath, FileName: f.Name(), Size: fi.Size(), // XXHash: xxHash.Sum64(), }) } } return formats } func lsEmbResources() { _ = fs.WalkDir(embResources, ".", func(path string, d fs.DirEntry, err error) error { c(err, "") // fmt.Println(" ", path) _ = path return nil }) } func copyEmbResources(destPath string) { _ = fs.WalkDir(embResources, ".", func(fsPath string, fsEntry fs.DirEntry, err error) error { destFsPath := strings.ReplaceAll(fsPath, "embResources/", "") if fsPath == "." || fsPath == "embResources" { return nil } else if fsEntry.IsDir() { err := os.MkdirAll(filepath.Join(destPath, destFsPath), 0755) c(err, "make new directory at the destination...") } else { newFile, err := os.Create(filepath.Join(destPath, destFsPath)) c(err, "create new file at the destination..") defer newFile.Close() embFile, err := embResources.Open(fsPath) _, err = io.Copy(newFile, embFile) c(err, "copy embResources/ file into the new file at the destination...") } return nil }) } func writeDataJs(calibrePath string, books []*BookJSON) { // j, _ := json.MarshalIndent(&BooksJSON{books}) dataJsFirst, err := os.Create(filepath.Join(calibrePath, "static", "data1.js")) c(err, "create dataJsFirst file...") defer dataJsFirst.Close() endB := 24 if endB >= len(books) { endB = len(books) } j1, _ := json.Marshal(&DataJs{ Portable: true, Total: len(books), Books: books[0:endB], }) calibreBooks1 := []byte("CALIBRE_BOOKS1=") _, _ = dataJsFirst.Write(append(calibreBooks1, j1...)) // [24:120], [120:216], [216:3288], [3288:6360], [6360:9432], [9432:12504] block := endB counter := 2 for i := range [2]int{} { ii := make([]int, int(math.Pow(2, float64(i+1)))) for j := range ii { _ = j var jsn []byte endBlock := block + int(math.Pow(32, float64(i+1)))*3 if endBlock > len(books) { endBlock = len(books) } dataJs, err := os.Create(filepath.Join(calibrePath, "static", fmt.Sprintf("data%d.js", counter))) c(err, "create dataJs file...") defer dataJs.Close() if endBlock <= len(books) { jsn, _ = json.Marshal(&DataJs{ Portable: true, Books: books[block:endBlock], }) } else { jsn, _ = json.Marshal(&DataJs{ Portable: true, Books: []*BookJSON{}, }) } calibreBooks := []byte(fmt.Sprintf("CALIBRE_BOOKS%d=", counter)) _, _ = dataJs.Write(append(calibreBooks, jsn...)) block = endBlock counter++ } } dataJsLast, err := os.Create(filepath.Join(calibrePath, "static", "data8.js")) c(err, "create dataJsLast file...") defer dataJsLast.Close() j8, _ := json.Marshal(&DataJs{ Portable: true, Total: len(books), Books: books[block:], }) calibreBooks8 := []byte("CALIBRE_BOOKS8=") _, _ = dataJsLast.Write(append(calibreBooks8, j8...)) } func PrintExePath() { fmt.Println(os.Executable()) } func RenderStandaloneApp(calibrePath, librarianName, libraryUUID, librarySecret, jsonPath string) { var outputs sync.Map var wg sync.WaitGroup count := 0 err := godirwalk.Walk(calibrePath, &godirwalk.Options{ Callback: func(path string, info *godirwalk.Dirent) error { if info.Name() == "metadata.opf" { bookOpf := &BookOpf{} count++ wg.Add(1) go func(bookOpf *BookOpf) { // randSuffix := uuid.New() // randSuffix := uuid.NewV4() relativeDirPath := strings.NewReplacer(calibrePath, "", "metadata.opf", "").Replace(path) f, _ := os.ReadFile(path) _ = xml.Unmarshal([]byte(f), &bookOpf) book := &BookJSON{ // Id: randSuffix.String(), Id: bookOpf.Uuid(librarySecret), Title: bookOpf.Metadata.Title, Librarian: librarianName, // LibraryUUID: randSuffix.String(), LibraryUUID: libraryUUID, TitleSort: bookOpf.TitleSort(), Authors: bookOpf.Authors(), Pubdate: bookOpf.Metadata.Published, LastModified: bookOpf.LastModified(), Tags: bookOpf.Tags(), Publisher: bookOpf.Metadata.Publisher, Abstract: bookOpf.Metadata.Description, Languages: bookOpf.Languages(), Identifiers: bookOpf.Identifiers(), Formats: Formats(calibrePath, relativeDirPath), CoverUrl: filepath.Join(relativeDirPath, "cover.jpg"), } // outputs.Store(fmt.Sprintf("%s_%s", book.LastModified, randSuffix.String()), book) outputs.Store(fmt.Sprintf("%s_%s", book.LastModified, uuid.NewV4().String()), book) wg.Done() }(bookOpf) } if count > 100 { wg.Wait() count = 0 } return nil }, ErrorCallback: func(path string, err error) godirwalk.ErrorAction { fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) return godirwalk.SkipNode }, Unsorted: true}) wg.Wait() c(err, "") var keysByDate []string outputs.Range(func(k, v interface{}) bool { keysByDate = append(keysByDate, k.(string)) return true }) sort.Sort(sort.Reverse(sort.StringSlice(keysByDate))) var books []*BookJSON for _, keyByDate := range keysByDate { b, _ := outputs.Load(keyByDate) books = append(books, b.(*BookJSON)) } lsEmbResources() copyEmbResources(calibrePath) writeDataJs(calibrePath, books) hugoCatalog := map[string]*BookJSON{} for _, book := range books { hugoCatalog[book.Id] = book } if jsonPath != "" { jsonDump, _ := json.Marshal(hugoCatalog) // _ = j err = os.WriteFile(jsonPath, jsonDump, 0666) c(err, "writing json file failed...") } fmt.Printf("Check out: %sBROWSE_LIBRARY.html", calibrePath) }