3ec7bb5222
Some checks failed
Build & Publish Docker Image / build-and-push (push) Failing after 11s
123 lines
3.2 KiB
Go
123 lines
3.2 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"emly-api-go/internal/models"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/jmoiron/sqlx"
|
|
)
|
|
|
|
func MigrateReportFilesToS3(db *sqlx.DB, s3conn *S3Connector, dbName string) error {
|
|
var wg sync.WaitGroup
|
|
errCh := make(chan error, 128) // buffer ragionevole
|
|
reportsRows, err := db.Query("SELECT id, created_at, updated_at FROM emly_bugreports_dev.bug_reports ORDER BY created_at DESC")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reportsRows.Close()
|
|
|
|
var totalReports, totalFiles, skipped, uploaded int
|
|
|
|
for reportsRows.Next() {
|
|
var reportId int
|
|
var createdAt, updatedAt time.Time
|
|
|
|
if err := reportsRows.Scan(
|
|
&reportId, &createdAt, &updatedAt,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
totalReports++
|
|
log.Printf("[migrate] processing report %d", reportId)
|
|
|
|
filesRows, err := db.Query(
|
|
"SELECT id, report_id, filename FROM emly_bugreports_dev.bug_report_files WHERE report_id = ?",
|
|
reportId,
|
|
)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for filesRows.Next() {
|
|
var fileID int
|
|
var fileReportID int
|
|
var fileName string
|
|
if err := filesRows.Scan(&fileID, &fileReportID, &fileName); err != nil {
|
|
filesRows.Close()
|
|
return err
|
|
}
|
|
|
|
var file models.BugReportFile
|
|
err := db.GetContext(context.Background(), &file, fmt.Sprintf("SELECT filename, mime_type, data FROM %s.bug_report_files WHERE report_id = ? AND id = ?", dbName), reportId, fileID)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
log.Printf("[migrate] report %d / file %d: not found in bug_report_files, skipping", reportId, fileID)
|
|
skipped++
|
|
continue
|
|
}
|
|
if err != nil {
|
|
filesRows.Close()
|
|
return fmt.Errorf("report %d / file %d: %w", reportId, fileID, err)
|
|
}
|
|
|
|
if s3conn != nil {
|
|
s3Key := fmt.Sprintf("emly-api-files/bug-reports/%d/files/%s", reportId, fileName)
|
|
dataCopy := make([]byte, len(file.Data))
|
|
copy(dataCopy, file.Data)
|
|
mime := file.MimeType
|
|
fname := file.Filename
|
|
totalFiles++
|
|
log.Printf("[migrate] report %d / file %d (%s, %d bytes): uploading to s3://%s", reportId, fileID, fname, len(dataCopy), s3Key)
|
|
wg.Add(1)
|
|
go func(key, mimeType, filename string, payload []byte, rid, fid int) {
|
|
defer wg.Done()
|
|
|
|
_, upErr := s3conn.UploadFile(
|
|
context.Background(),
|
|
key,
|
|
bytes.NewReader(payload),
|
|
mimeType,
|
|
map[string]string{"filename": filename},
|
|
)
|
|
if upErr != nil {
|
|
errCh <- fmt.Errorf("report %d / file %d (%s): %w", rid, fid, key, upErr)
|
|
log.Printf("[migrate] [ERROR] upload failed for s3://%s: %v", key, upErr)
|
|
return
|
|
}
|
|
log.Printf("[migrate] upload complete: s3://%s", key)
|
|
}(s3Key, mime, fname, dataCopy, reportId, fileID)
|
|
|
|
uploaded++
|
|
}
|
|
}
|
|
|
|
if err := filesRows.Close(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
wg.Wait()
|
|
close(errCh)
|
|
|
|
var uploadErrCount int
|
|
for e := range errCh {
|
|
uploadErrCount++
|
|
log.Printf("[migrate] [ERROR] %v", e)
|
|
}
|
|
|
|
log.Printf("[migrate] done — reports: %d, files queued: %d, skipped: %d, upload errors: %d",
|
|
totalReports, uploaded, skipped, uploadErrCount)
|
|
|
|
if uploadErrCount > 0 {
|
|
return fmt.Errorf("migration completed with %d upload errors", uploadErrCount)
|
|
}
|
|
return nil
|
|
}
|