aboutsummaryrefslogtreecommitdiff
path: root/tasks/producersync.task.go
diff options
context:
space:
mode:
Diffstat (limited to 'tasks/producersync.task.go')
-rw-r--r--tasks/producersync.task.go282
1 files changed, 48 insertions, 234 deletions
diff --git a/tasks/producersync.task.go b/tasks/producersync.task.go
index e211eb3..59864ce 100644
--- a/tasks/producersync.task.go
+++ b/tasks/producersync.task.go
@@ -1,264 +1,78 @@
package tasks
import (
- "fmt"
- "metachan/database"
"metachan/entities"
+ "metachan/repositories"
"metachan/utils/api/jikan"
"metachan/utils/logger"
"time"
)
func ProducerSync() error {
- logger.Log("Starting producer sync (includes studios and licensors)...", logger.LogOptions{
- Level: logger.Info,
- Prefix: "ProducerSync",
- })
+ logger.Infof("ProducerSync", "Starting producer sync (includes studios and licensors)")
- client := jikan.NewJikanClient()
- page := 1
- totalFetched := 0
- var totalPages int
- var totalProducers int
- startTime := time.Now()
+ response, err := jikan.GetAnimeProducers()
+ if err != nil {
+ logger.Errorf("ProducerSync", "Failed to fetch producers: %v", err)
+ return err
+ }
- for {
- logger.Log(fmt.Sprintf("Fetching producers page %d...", page), logger.LogOptions{
- Level: logger.Info,
- Prefix: "ProducerSync",
- })
+ total := len(response.Data)
+ logger.Infof("ProducerSync", "Fetched %d producers from MAL", total)
+
+ startTime := time.Now()
- response, err := client.GetAnimeProducers(page)
+ for i, producerData := range response.Data {
+ producerDetail, err := jikan.GetProducerByID(producerData.MALID)
if err != nil {
- logger.Log(fmt.Sprintf("Failed to fetch producers page %d: %v", page, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- // If we fetched at least one page, continue with what we have
- if page > 1 {
- break
+ logger.Warnf("ProducerSync", "Failed to fetch details for producer %d: %v", producerData.MALID, err)
+ continue
+ }
+
+ var imageID *uint
+ if producerDetail.Data.Images.JPG.ImageURL != "" {
+ image := entities.SimpleImage{
+ ImageURL: producerDetail.Data.Images.JPG.ImageURL,
+ }
+ id, err := repositories.CreateOrUpdateSimpleImage(&image)
+ if err == nil {
+ imageID = &id
}
- return err
}
- if len(response.Data) == 0 {
- break
+ producer := entities.Producer{
+ MALID: producerDetail.Data.MALID,
+ URL: producerDetail.Data.URL,
+ Favorites: producerDetail.Data.Favorites,
+ Count: producerDetail.Data.Count,
+ Established: producerDetail.Data.Established,
+ About: producerDetail.Data.About,
+ ImageID: imageID,
}
- // Set total pages from first response
- if page == 1 {
- totalPages = response.Pagination.LastVisiblePage
- totalProducers = totalPages * len(response.Data)
- logger.Log(fmt.Sprintf("Total pages: %d, Estimated producers: %d", totalPages, totalProducers), logger.LogOptions{
- Level: logger.Info,
- Prefix: "ProducerSync",
+ for _, title := range producerDetail.Data.Titles {
+ producer.Titles = append(producer.Titles, entities.SimpleTitle{
+ Type: title.Type,
+ Title: title.Title,
})
}
- // Process each producer
- for _, producerData := range response.Data {
- // Check if producer already exists
- var existingProducer entities.Producer
- result := database.DB.Where("mal_id = ?", producerData.MALID).First(&existingProducer)
-
- if result.Error != nil {
- // Producer doesn't exist, create new
- producer := entities.Producer{
- MALID: producerData.MALID,
- URL: producerData.URL,
- Favorites: producerData.Favorites,
- Count: producerData.Count,
- Established: producerData.Established,
- About: producerData.About,
- }
-
- // Create producer in database
- if err := database.DB.Create(&producer).Error; err != nil {
- logger.Log(fmt.Sprintf("Failed to create producer %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- continue
- }
-
- // Add titles
- for _, title := range producerData.Titles {
- producerTitle := entities.ProducerTitle{
- ProducerID: producer.ID,
- Type: title.Type,
- Title: title.Title,
- }
- if err := database.DB.Create(&producerTitle).Error; err != nil {
- logger.Log(fmt.Sprintf("Failed to create producer title for %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- }
- }
-
- // Add image
- if producerData.Images.JPG.ImageURL != "" {
- producerImage := entities.ProducerImage{
- ProducerID: producer.ID,
- ImageURL: producerData.Images.JPG.ImageURL,
- }
- if err := database.DB.Create(&producerImage).Error; err != nil {
- logger.Log(fmt.Sprintf("Failed to create producer image for %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- }
- }
-
- // Fetch and add external URLs
- time.Sleep(350 * time.Millisecond) // Rate limiting
- externalResp, err := client.GetProducerExternal(producerData.MALID)
- if err != nil {
- logger.Log(fmt.Sprintf("Failed to fetch external URLs for producer %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- } else {
- for _, ext := range externalResp.Data {
- producerExt := entities.ProducerExternalURL{
- ProducerID: producer.ID,
- Name: ext.Name,
- URL: ext.URL,
- }
- if err := database.DB.Create(&producerExt).Error; err != nil {
- logger.Log(fmt.Sprintf("Failed to create external URL for producer %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- }
- }
- }
-
- // Get primary title (default or first available)
- primaryTitle := "Unknown"
- for _, title := range producerData.Titles {
- if title.Type == "Default" {
- primaryTitle = title.Title
- break
- }
- }
- if primaryTitle == "Unknown" && len(producerData.Titles) > 0 {
- primaryTitle = producerData.Titles[0].Title
- }
-
- logger.Log(fmt.Sprintf("Created producer: %s (ID: %d, Count: %d)", primaryTitle, producerData.MALID, producerData.Count), logger.LogOptions{
- Level: logger.Success,
- Prefix: "ProducerSync",
- })
- } else {
- // Producer exists, update it
- existingProducer.URL = producerData.URL
- existingProducer.Favorites = producerData.Favorites
- existingProducer.Count = producerData.Count
- existingProducer.Established = producerData.Established
- existingProducer.About = producerData.About
-
- if err := database.DB.Save(&existingProducer).Error; err != nil {
- logger.Log(fmt.Sprintf("Failed to update producer %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- continue
- }
-
- // Delete and recreate titles
- database.DB.Where("producer_id = ?", existingProducer.ID).Delete(&entities.ProducerTitle{})
- for _, title := range producerData.Titles {
- producerTitle := entities.ProducerTitle{
- ProducerID: existingProducer.ID,
- Type: title.Type,
- Title: title.Title,
- }
- database.DB.Create(&producerTitle)
- }
-
- // Update image
- var existingImage entities.ProducerImage
- if database.DB.Where("producer_id = ?", existingProducer.ID).First(&existingImage).Error == nil {
- existingImage.ImageURL = producerData.Images.JPG.ImageURL
- database.DB.Save(&existingImage)
- } else if producerData.Images.JPG.ImageURL != "" {
- producerImage := entities.ProducerImage{
- ProducerID: existingProducer.ID,
- ImageURL: producerData.Images.JPG.ImageURL,
- }
- database.DB.Create(&producerImage)
- }
-
- // Update external URLs
- time.Sleep(350 * time.Millisecond) // Rate limiting
- externalResp, err := client.GetProducerExternal(producerData.MALID)
- if err != nil {
- logger.Log(fmt.Sprintf("Failed to fetch external URLs for producer %d: %v", producerData.MALID, err), logger.LogOptions{
- Level: logger.Error,
- Prefix: "ProducerSync",
- })
- } else {
- database.DB.Where("producer_id = ?", existingProducer.ID).Delete(&entities.ProducerExternalURL{})
- for _, ext := range externalResp.Data {
- producerExt := entities.ProducerExternalURL{
- ProducerID: existingProducer.ID,
- Name: ext.Name,
- URL: ext.URL,
- }
- database.DB.Create(&producerExt)
- }
- }
-
- primaryTitle := "Unknown"
- for _, title := range producerData.Titles {
- if title.Type == "Default" {
- primaryTitle = title.Title
- break
- }
- }
- if primaryTitle == "Unknown" && len(producerData.Titles) > 0 {
- primaryTitle = producerData.Titles[0].Title
- }
-
- logger.Log(fmt.Sprintf("Updated producer: %s (ID: %d, Count: %d)", primaryTitle, producerData.MALID, producerData.Count), logger.LogOptions{
- Level: logger.Success,
- Prefix: "ProducerSync",
- })
- }
-
- totalFetched++
-
- // Progress update every 10 producers
- if totalFetched%10 == 0 && totalProducers > 0 {
- progress := float64(totalFetched) / float64(totalProducers) * 100
- elapsed := time.Since(startTime)
- avgTimePerProducer := elapsed / time.Duration(totalFetched)
- remaining := totalProducers - totalFetched
- eta := avgTimePerProducer * time.Duration(remaining)
-
- logger.Log(fmt.Sprintf("Progress: %d/%d - %.1f%% | ETA: %v", totalFetched, totalProducers, progress, eta.Round(time.Second)), logger.LogOptions{
- Level: logger.Info,
- Prefix: "ProducerSync",
- })
- }
-
- time.Sleep(350 * time.Millisecond) // Rate limiting between producers
+ for _, ext := range producerDetail.Data.External {
+ producer.ExternalURLs = append(producer.ExternalURLs, entities.ExternalURL{
+ Name: ext.Name,
+ URL: ext.URL,
+ })
}
- // Check if there's more data
- if !response.Pagination.HasNextPage {
- break
+ if err := repositories.CreateOrUpdateProducer(&producer); err != nil {
+ logger.Warnf("ProducerSync", "Failed to sync producer %d: %v", producerData.MALID, err)
+ continue
}
- page++
- time.Sleep(1 * time.Second) // Additional delay between pages
+ progress, eta := calculateProgress(i+1, total, startTime)
+ logger.Infof("ProducerSync", "Progress: %d/%d (%.1f%%) | ETA: %v", i+1, total, progress, eta)
}
- logger.Log(fmt.Sprintf("Producer sync completed successfully. Total: %d producers", totalFetched), logger.LogOptions{
- Level: logger.Success,
- Prefix: "ProducerSync",
- })
-
+ logger.Successf("ProducerSync", "Producer sync completed. Total: %d producers", total)
return nil
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-06 01:35:14 +0000