diff --git a/.env.example b/.env.example index 53dd7e6..156194e 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,12 @@ -couch_db_source=http://admin:adminadmin@192.168.1.2:5984/ +couch_db_source= db_trendyol=trendyol_db_v2 db_ty_categories=ty_categories -remote_url=https://sarga.com.tm/api/ +remote_url= descendant_category_url=descendant-categories -db=test_baza -du=merdan -dp=qazwsx12 -da=localhost -dp=3306 -#database_url ="merdan:qazwsx12@tcp(127.0.0.1:3306)/test_baza?parseTime=true" -database_url ="orient:orient@tcp(192.168.1.2:3306)/bagisto_sarga?parseTime=true" -couchdb_url = "http://admin:adminadmin@192.168.1.2:5984/" -scout_flash = "http://sarga.com.tm/scrap/scoute-flush" \ No newline at end of file +database_url= +couchdb_url= +scout_flash= +port= \ No newline at end of file diff --git a/controllers/ImportController.go b/controllers/ImportController.go index f6f0809..74805ca 100644 --- a/controllers/ImportController.go +++ b/controllers/ImportController.go @@ -1065,36 +1065,3 @@ func UpdateProduct(product models.Product, db *gorm.DB, productFlat gm.ProductFl return errFlat } - -//func productAttributesAndFlat(data *models.Product) ([]gm.ProductAttributeValue,gm.ProductFlat){ -// -// var description string -// -// for _, desc := range data.Descriptions { -// description += "

" + desc.Description + "

" -// } -// -// weight, _ := strconv.ParseFloat(data.Weight, 64) -// -// flat := gm.ProductFlat{ -// Status: true, -// VisibleIndividually: true, -// Name: data.Name, -// Sku: data.ProductGroupID, -// ProductNumber: data.ProductNumber, -// Description: description, -// UrlKey: data.ProductGroupID, -// Weight: weight, -// FavoritesCount: uint(data.FavoriteCount), -// } -// return []gm.ProductAttributeValue{ -// {AttributeID: AttributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount}, -// {AttributeID: AttributesMap["source"].ID, TextValue: data.URLKey}, -// {AttributeID: AttributesMap["product_number"].ID, TextValue: data.ProductNumber}, -// {AttributeID: AttributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"}, -// {AttributeID: AttributesMap["weight"].ID, TextValue: data.Weight}, -// {AttributeID: AttributesMap["status"].ID, BooleanValue: true}, -// {AttributeID: AttributesMap["visible_individually"].ID, BooleanValue: true}, -// {AttributeID: AttributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"}, -// },flat -//} diff --git a/controllers/ImportV2.go b/controllers/ImportV2.go index 8a14748..80290ba 100644 --- a/controllers/ImportV2.go +++ b/controllers/ImportV2.go @@ -46,6 +46,7 @@ func StartProductImport(w http.ResponseWriter, _ *http.Request) { } importRepo.ImportWGroup.Wait() + //todo delete galan wishlist //scout index flush if err = gm.Flush(); err != nil { diff --git a/controllers/ParseController.go b/controllers/ParseController.go index fbcbe40..18294d0 100644 --- a/controllers/ParseController.go +++ b/controllers/ParseController.go @@ -1,29 +1,82 @@ package controller import ( - gm "db_service/gorm_models" + "db_service/models" helper "db_service/pkg" "db_service/repositories" "encoding/json" - "errors" - "gorm.io/driver/mysql" - "gorm.io/gorm" + "fmt" "log" "net/http" - "os" "strconv" + "strings" "time" ) func ParseLink(w http.ResponseWriter, route *http.Request) { - start := time.Now() link := route.URL.Query().Get("url") + helper.Info("link: ", link) + if strings.Contains(link, "lcwaikiki.com"){ + ParseLinkLCW(link,w) + }else { + ParseLinkTY(link,w) + } + + +} + +func ParseLinkLCW(link string,w http.ResponseWriter) { + start := time.Now() + fmt.Println(link) + linkParser := repositories.NewLCWScraper(link) + + product, err := linkParser.InitProductDetailParsing() + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{ + "msg": err.Error(), + }) + fmt.Println(err) + return + } + + ///////////////////////////// + importRepo, err := repositories.ParseImporterInstance() + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + fmt.Println(err) + return + } + //wait until initialization data is loaded to memory + importRepo.ImportWGroup.Wait() + log.Println(product); + + if err = importRepo.UpdateOrCreateLCW(product).Error; err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + fmt.Println(err) + return + } + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]string{ + "msg": "Link parsed successfully", + "productGroupId": product.ProductNumber, + }) + + elapsed := time.Since(start) + log.Printf("end parse took %s", elapsed) + return +} + +func ParseLinkTY(link string,w http.ResponseWriter) { + start := time.Now() + linkParser := repositories.NewLinkParser(link) - //todo product, err := linkParser.ParseLink() if err != nil { @@ -31,70 +84,75 @@ func ParseLink(w http.ResponseWriter, route *http.Request) { json.NewEncoder(w).Encode(map[string]string{ "msg": err.Error(), }) - - return - } - - jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID) - - if err != nil { - helper.Error(err) - return - } - - baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{}) - - if err != nil { - helper.Error(err) - return - } - - var productFlat gm.ProductFlat - - err = baza.Preload("Variants"). - //Preload("Product"). - //Preload("Proudct.AttributeValues","attribute_id in(11,13)"). - First(&productFlat, "sku = ?", jsonProduct.ProductGroupID).Error - - if err != nil && errors.Is(err, gorm.ErrRecordNotFound) { - helper.Error(err) - - if len(AttributesMap) == 0 { - var attributes, err = gm.GetAttributes(baza) - - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - for _, atrattribute := range attributes { - AttributesMap[atrattribute.Code] = atrattribute - } - } - - err = ImportProduct(jsonProduct, baza) - } else if err == nil { - err = UpdateProduct(jsonProduct, baza, productFlat) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(map[string]string{ - "msg": "Link parsed successfully", - "productGroupId": strconv.Itoa(product.ProductGroupID), - }) - elapsed := time.Since(start) - log.Printf("end parse took %s", elapsed) - return - } - - if err != nil { - helper.Error(err) + fmt.Println(err) return } elapsed := time.Since(start) - log.Printf("end parse took %s", elapsed) + log.Printf("link parsed at %s", elapsed) + jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID) + + elapsed = time.Since(start) + log.Printf("product gat at %s", elapsed) + + + if err != nil { + helper.Error(err) + return + } + ///////////////////////////// + importRepo, err := repositories.ImporterInstance() + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + //wait until initialization data is loaded to memory + importRepo.ImportWGroup.Wait() + + if err = importRepo.UpdateOrCreate(jsonProduct).Error; err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(map[string]string{ "msg": "Link parsed successfully", - "productGroupId": strconv.Itoa(product.ProductGroupID), + "productGroupId": strconv.Itoa(product.ID), }) + elapsed = time.Since(start) + log.Printf("imported at %s", elapsed) + return + ///////////////////////////////////////// } + +func ImportLink(w http.ResponseWriter, route *http.Request) { + + var product models.Product + + err := json.NewDecoder(route.Body).Decode(&product) + + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + ///////////////////////////// + importRepo, err := repositories.ImporterInstance() + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + //wait until initialization data is loaded to memory + importRepo.ImportWGroup.Wait() + + if err = importRepo.UpdateOrCreate(product).Error; err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + + json.NewEncoder(w).Encode(map[string]string{ + "msg": "Link parsed successfully", + "productGroupId": product.ProductGroupID, + }) +} diff --git a/go.mod b/go.mod index ca0a144..1a275fd 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module db_service go 1.19 require ( + github.com/PuerkitoBio/goquery v1.8.0 + github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24 + github.com/chromedp/chromedp v0.8.7 github.com/gorilla/mux v1.8.0 github.com/gosimple/slug v1.12.0 github.com/joho/godotenv v1.4.0 @@ -12,8 +15,17 @@ require ( ) require ( + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/chromedp/sysutil v1.0.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.1.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect + golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect ) diff --git a/go.sum b/go.sum index 18b7db5..3411b91 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,21 @@ +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24 h1:IKBU366KowxMcDpu9cgYEaXsiO0eBfPosTS0Krqfb8M= +github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.8.7 h1:dYOYc5ynTBzwSLOi+1IfgHwPr8r2BqV48l/RC+3OuJ0= +github.com/chromedp/chromedp v0.8.7/go.mod h1:iL+ywnwk3eG3EVXV1ackXBMNzdEh3Ye/KHvQkq1KRKU= +github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gosimple/slug v1.12.0 h1:xzuhj7G7cGtd34NXnW/yF0l+AGNfWqwgh/IXgFy7dnc= @@ -13,8 +29,24 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc= github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY= gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= diff --git a/gorm_models/brand.go b/gorm_models/brand.go index 7f03085..1bf9b05 100644 --- a/gorm_models/brand.go +++ b/gorm_models/brand.go @@ -26,14 +26,21 @@ func FindOrCreateBrand(db *gorm.DB, brand string, categories []Category) (Brand, code := slug.Make(brand) - err := db.Omit("Categories").FirstOrCreate(&brandObject, Brand{Name: brand, Code: code, Categories: categories}).Error - // err := db.Model(&Brand{}).Find(&brandObject).Error + err := db.FirstOrCreate(&brandObject, Brand{Name: brand, Code: code}).Error + //err := db.Model(Brand{Name: brand, Code: code}).First(&brandObject).Error + //log.Println(Brand{Name: brand, Code: code}) if err != nil { + //err := db.Omit("Categories.*").FirstOrCreate(&brandObject, Brand{Name: brand, Code: code, Categories: categories}).Error log.Println("ERR0000000000000001" + err.Error()) return brandObject, err } + db.Model(&brandObject).Association("Categories").Append(categories) + + return brandObject, nil + + } // var brandObject Brand diff --git a/gorm_models/family.go b/gorm_models/family.go index d698c2d..9028d94 100644 --- a/gorm_models/family.go +++ b/gorm_models/family.go @@ -1,16 +1,25 @@ package gorm_models import ( + "errors" "log" "gorm.io/gorm" ) type AttributeOption struct { - ID uint `gorm:"primaryKey"` - AttributeID uint - AdminName string - SortOrder int `sql:"DEFAULT:NULL" gorm:"default:null"` + ID uint `gorm:"primaryKey"` + AttributeID uint + AdminName string + SortOrder int `sql:"DEFAULT:NULL" gorm:"default:null"` + Translations []AttributeOptionTranslation +} + +type AttributeOptionTranslation struct { + ID uint `gorm:"primaryKey"` + AttributeOptionID uint + Locale string + Label string } type Attribute struct { @@ -61,11 +70,27 @@ func GetAttributes(db *gorm.DB) ([]Attribute, error) { func GetAttributeOption(db *gorm.DB, attrbuteID uint, value string) AttributeOption { var option AttributeOption - err := db.FirstOrCreate(&option, AttributeOption{AttributeID: attrbuteID, AdminName: value}).Error - if err != nil { - log.Println(err.Error()) + err := db.Where("attribute_id=? and admin_name=?", attrbuteID, value).First(&option).Error + + if err != nil && errors.Is(err, gorm.ErrRecordNotFound) { + option = AttributeOption{ + AttributeID: attrbuteID, + AdminName: value, + SortOrder: 1000, + Translations: []AttributeOptionTranslation{ + {Locale: "tm", Label: value}, + {Locale: "ru", Label: value}, + {Locale: "tr", Label: value}, + }, + } + + if err := db.Create(&option).Error; err != nil { + log.Println(err.Error()) + + } } + return option } diff --git a/gorm_models/product.go b/gorm_models/product.go index bd88456..79161bb 100644 --- a/gorm_models/product.go +++ b/gorm_models/product.go @@ -4,6 +4,7 @@ import ( "context" helper "db_service/pkg" "gorm.io/gorm" + "log" "os" "time" ) @@ -32,13 +33,12 @@ type ProductRelation struct { } type ProductFlat struct { - ID uint `gorm:"primary_key"` - Sku string - ProductNumber string `sql:"DEFAULT:NULL" gorm:"default:null"` - Name string `sql:"DEFAULT:''" gorm:"default:''"` - Weight float64 `sql:"DEFAULT:NULL" gorm:"type:decimal(12,4);default:null"` - Status bool `sql:"DEFAULT:NULL" gorm:"default:null"` - //Source string `sql:"DEFAULT:NULL" gorm:"default:null"` + ID uint `gorm:"primary_key"` + Sku string + ProductNumber string `sql:"DEFAULT:NULL" gorm:"default:null"` + Name string `sql:"DEFAULT:''" gorm:"default:''"` + Weight float64 `sql:"DEFAULT:NULL" gorm:"type:decimal(12,4);default:null"` + Status bool `sql:"DEFAULT:NULL" gorm:"default:null"` VisibleIndividually bool `sql:"DEFAULT:NULL" gorm:"default:null"` Price float64 `gorm:"type:decimal(12,4)"` MinPrice float64 `gorm:"type:decimal(12,4)"` @@ -60,10 +60,14 @@ type ProductFlat struct { ColorLabel string `sql:"DEFAULT:NULL" gorm:"default:null"` Size int `sql:"DEFAULT:NULL" gorm:"default:null"` SizeLabel string `sql:"DEFAULT:NULL" gorm:"default:null"` + Boyut int `sql:"DEFAULT:NULL" gorm:"default:null"` + BoyutLabel string `sql:"DEFAULT:NULL" gorm:"default:null"` MetaTitle string `sql:"DEFAULT:NULL" gorm:"default:null"` MetaKeywords string `sql:"DEFAULT:NULL" gorm:"default:null"` BrandID uint `sql:"DEFAULT:NULL" gorm:"default:null"` Brand Brand + Cinsiyet int `sql:"DEFAULT:NULL" gorm:"default:null"` + CinsiyetLabel string `sql:"DEFAULT:NULL" gorm:"default:null"` } type ProductImage struct { @@ -104,10 +108,43 @@ func DeleteProducts(db *gorm.DB) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() //qb := "DELETE FROM products WHERE id NOT IN (select product_id as id from wishlist) AND id NOT IN (select product_id as id from order_items) AND id NOT IN (select parent_id as idfrom order_items);" - qb := "DELETE p FROM products p LEFT JOIN order_items oi ON p.id = oi.product_id LEFT JOIN order_items op ON p.id = op.parent_id WHERE oi.id IS NULL AND op.id IS NULL;" - return db.WithContext(ctx).Exec(qb).Error + qb := "DELETE p FROM products p " + + "LEFT JOIN order_items oi ON p.id = oi.product_id " + + "LEFT JOIN order_items op ON p.id = op.parent_id " + + "LEFT JOIN wishlist wp ON p.id = wp.product_id " + + "LEFT JOIN wishlist wl ON wl.product_id = p.parent_id " + + "JOIN marketplace_products mp ON p.id = mp.product_id "+ + "WHERE oi.id IS NULL AND op.id IS NULL AND wp.id IS NULL AND wl.id IS NULL AND mp.marketplace_seller_id=1;" + db.WithContext(ctx).Exec(qb) + //db.WithContext(ctx).Exec("UPDATE product_flat set sku=concat(id,\"-ordered\"), status=0 where status=1 AND product_id IN (SELECT product_id from marketplace_products where marketplace_seller_id=1)" ) + //db.WithContext(ctx).Exec("UPDATE products set sku=concat(id,\"-ordered\") WHERE id IN (SELECT product_id from marketplace_products where marketplace_seller_id=1)") + return db.Error } +func GetWishlistProducts(db *gorm.DB) ([]Product, error){ + var products [] Product + err := db.Joins("JOIN wishlist wp ON products.id = wp.product_id").Joins("JOIN marketplace_products mp ON products.id = mp.product_id where marketplace_seller_id=1").Find(&products).Error + + if err != nil { + log.Println(err.Error()) + return nil, err + } + + return products, nil + //qb := "SELECT p.id,p.sku FROM products p " + + // "JOIN wishlist wp ON p.id = wp.product_id " + + // "JOIN marketplace_products mp ON p.id = mp.product_id where marketplace_seller_id=1);" +} + +//func DisableProducts (db *gorm.DB) error { +// ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) +// defer cancel() +// db.WithContext(ctx).Exec("UPDATE product_flat set sku=concat(sku,\"-ordered\"), status=0") +// db.WithContext(ctx).Exec("UPDATE products set sku=concat(sku,\"-ordered\")") +// return db.Error +//} + + func Flush() error { _, err := helper.SendRequest("GET", os.Getenv("scout_flash"), nil, "") diff --git a/gorm_models/vendor.go b/gorm_models/vendor.go index f630f56..8952e35 100644 --- a/gorm_models/vendor.go +++ b/gorm_models/vendor.go @@ -11,6 +11,7 @@ type MarketplaceProduct struct { CreatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"` UpdatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"` ProductID uint + //Product Product ParentID *uint `sql:"DEFAULT:NULL" gorm:"default:null"` Condition string `sql:"DEFAULT:NULL" gorm:"default:null"` Price float64 diff --git a/gorm_models/wishlist.go b/gorm_models/wishlist.go new file mode 100644 index 0000000..396a725 --- /dev/null +++ b/gorm_models/wishlist.go @@ -0,0 +1,11 @@ +package gorm_models + +import "time" + +type Wishlist struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"` + UpdatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"` + ProductID uint + //Product Product +} diff --git a/main.go b/main.go index 57e821f..0747bd9 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ func main() { //route.HandleFunc("/init-importer", controller.StartImport) route.HandleFunc("/init-importer", controller.StartProductImport) route.HandleFunc("/parse-link", controller.ParseLink) + route.HandleFunc("/import-link", controller.ImportLink).Methods("POST", "GET") err := http.ListenAndServe(os.Getenv("port"), route) if err != nil { diff --git a/models/lcw_scrap_data_model.go b/models/lcw_scrap_data_model.go new file mode 100644 index 0000000..903d969 --- /dev/null +++ b/models/lcw_scrap_data_model.go @@ -0,0 +1,17 @@ +package models + + +type LCWScrapDataModel struct { + Id string + Code string + Name string + OldPrice string + Price string + SizeVariants []SizeVariants + ColorOptions []LCWScrapDataModel +} + +type DescriptionModel struct { + Description string `json:"description"` + Bold bool `json:"bold"` +} diff --git a/models/product.go b/models/product.go index d537e85..26829a3 100644 --- a/models/product.go +++ b/models/product.go @@ -32,20 +32,16 @@ type Product struct { } type Price struct { - ProfitMargin int `json:"profitMargin"` - DiscountedPrice struct { - Text string `json:"text"` - Value float64 `json:"value"` - } `json:"discountedPrice"` - SellingPrice struct { - Text string `json:"text"` - Value float64 `json:"value"` - } `json:"sellingPrice"` - OriginalPrice struct { - Text string `json:"text"` - Value float64 `json:"value"` - } `json:"originalPrice"` - Currency string `json:"currency"` + ProfitMargin int `json:"profitMargin"` + DiscountedPrice PriceValue `json:"discountedPrice"` + SellingPrice PriceValue `json:"sellingPrice"` + OriginalPrice PriceValue `json:"originalPrice"` + Currency string `json:"currency"` +} + +type PriceValue struct { + Text string `json:"text"` + Value float64 `json:"value"` } type Variant struct { @@ -163,7 +159,7 @@ type TrendyolProductDetailModel struct { ProductCode string `json:"productCode"` Name string `json:"name"` NameWithProductCode string `json:"nameWithProductCode"` - Description string `json:"description"` + //Description string `json:"description"` ContentDescriptions []struct { Description string `json:"description"` Bold bool `json:"bold"` diff --git a/models/size_variant_model.go b/models/size_variant_model.go new file mode 100644 index 0000000..fe0b94e --- /dev/null +++ b/models/size_variant_model.go @@ -0,0 +1,7 @@ +package models + +type SizeVariants struct { + Price float64 `json:"price"` + Size string `json:"size"` + Stock int `json:"stock"` +} diff --git a/pkg/helper.go b/pkg/helper.go index 85a67f2..2219186 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -2,6 +2,7 @@ package helper import ( "context" + "db_service/models" "log" "net" "net/http" @@ -84,3 +85,28 @@ func GetCategoryWeight(slug string) string { return doc["weight"].(string) } +func IsLCWSizeVariantsAdded(items []models.Variant, keyItem models.Variant) bool { + for _, option := range items { + if option.ItemNumber == keyItem.ItemNumber { + return true + } + } + return false +} + +func IsImageAdded(images []string, image string) bool { + for _, img := range images { + if img == image { + return true + } + } + return false +} +func IsVariantsAdded(variants []models.Variant, itemNumber int) bool { + for _, variant := range variants { + if variant.ItemNumber == itemNumber { + return true + } + } + return false +} \ No newline at end of file diff --git a/pkg/request.go b/pkg/request.go index 43bf74f..dc6bd75 100644 --- a/pkg/request.go +++ b/pkg/request.go @@ -12,8 +12,8 @@ import ( func SendRequest(method string, endpoint string, values []byte, authKey string) ([]byte, error) { - const ConnectMaxWaitTime = 30 * time.Second - const RequestMaxWaitTime = 60 * time.Second + const ConnectMaxWaitTime = time.Minute + const RequestMaxWaitTime = 5 * time.Minute client := http.Client{ Transport: &http.Transport{ diff --git a/repositories/ImportRepository.go b/repositories/ImportRepository.go index f60b1a9..7f70fa1 100644 --- a/repositories/ImportRepository.go +++ b/repositories/ImportRepository.go @@ -22,6 +22,7 @@ type Importer struct { baza *gorm.DB families []gm.AttributeFamily sellers map[string]gm.MarketplaceSeller + //wishlist map[string]gm.Product AttributesMap map[string]gm.Attribute Error error ImportWGroup sync.WaitGroup @@ -30,7 +31,10 @@ type Importer struct { ColorMutex sync.Mutex SexMutex sync.Mutex } - +type GormErr struct { + Number int `json:"Number"` + Message string `json:"Message"` +} func ImporterInstance() (instance *Importer, err error) { db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true}) @@ -48,6 +52,7 @@ func ImporterInstance() (instance *Importer, err error) { go func(db *gorm.DB) { defer instance.ImportWGroup.Done() instance.mainCategories, instance.Error = gm.GetMainCategories(db) + }(db) //load families to memory @@ -113,6 +118,105 @@ func ImporterInstance() (instance *Importer, err error) { } }() + //load wishlist to memory + //go func(){ + // defer instance.ImportWGroup.Done() + // + // var wishlist, err = gm.GetWishlistProducts(db) + // + // if err != nil { + // instance.Error = err + // return + // } + // + // instance.wishlist = make(map[string]gm.Product, len(wishlist)) + // + // for _, product := range wishlist { + // instance.wishlist[product.Sku] = product + // } + //}() + + if instance.Error != nil { + log.Println(instance.Error) + return nil, instance.Error + } + + return instance, nil +} + +func ParseImporterInstance() (instance *Importer, err error) { + db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true}) + + if err != nil { + log.Println(err) + return nil, err + } + + instance = &Importer{baza: db} + instance.ImportWGroup.Add(3) + //load families to memory + go func() { + defer instance.ImportWGroup.Done() + instance.families, instance.Error = gm.GetFamilies(db) + }() + + //load attributes to memory + go func() { + defer instance.ImportWGroup.Done() + + if attributes, err := gm.GetAttributes(db); err != nil { + instance.Error = err + return + } else { + instance.AttributesMap = make(map[string]gm.Attribute, len(attributes)) + + for _, attribute := range attributes { + instance.AttributesMap[attribute.Code] = attribute + } + } + + if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil { + instance.Error = err + return + } else { + instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions)) + + for _, option := range colorOptions { + instance.ColorOptions[option.AdminName] = option + } + } + + if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil { + instance.Error = err + return + } else { + instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions)) + + for _, option := range sexOPtions { + instance.SexOptions[option.AdminName] = option + } + } + + }() + + //load sellers to memory + go func() { + defer instance.ImportWGroup.Done() + + var vendors, err = gm.GetSellers(db) + + if err != nil { + instance.Error = err + return + } + //init sellers map + instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors)) + + for _, vendor := range vendors { + instance.sellers[vendor.Url] = vendor + } + }() + if instance.Error != nil { log.Println(instance.Error) return nil, instance.Error @@ -237,7 +341,7 @@ func (importer *Importer) ImportProduct(product models.Product) (instance *Impor } if len(linkedProducts) > 1 { - //todo link products + var relation []gm.ProductRelation for index, variant := range linkedProducts { //spoint := "color" + strconv.Itoa(index) @@ -265,6 +369,13 @@ func (importer *Importer) ImportProduct(product models.Product) (instance *Impor } func (importer *Importer) importVariant(product models.Product) (*gm.Product, error) { + + // check if wishlisted then update if. + //if _, ok := importer.wishlist[product.ProductNumber]; ok { + // delete(importer.wishlist,product.ProductNumber) + // return importer.updateVariant(product) + //} + productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) if categories, err := gm.GetCatKeywords(importer.baza, product.Categories); err != nil { @@ -285,14 +396,46 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er tx := importer.baza.Begin() if err := tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil { - tx.Rollback() - return nil, err + //todo update categories + byteErr, _ := json.Marshal(err) + var newError GormErr + + if err1:= json.Unmarshal((byteErr), &newError);err1!=nil{ + tx.Rollback() + log.Println(err1,"err2") + return nil, err1 + } + + if newError.Number == 1062{ + var barProduct gm.Product + if err2 := tx.First(&barProduct,"sku = ?",mainPorduct.Sku).Error;err2 !=nil { + tx.Rollback() + log.Println(err2,"err3") + return nil, err2 + } + + if err3:= tx.Model(&barProduct).Association("Categories").Append(mainPorduct.Categories); err3!=nil{ + tx.Rollback() + log.Println(err3,"err4") + return nil, err3 + } + + if err4 := tx.Commit().Error; err4 ==nil{ + return importer.updateVariant(product) + } + }else{ + tx.Rollback() + log.Println(err,"er1") + return nil, err + } + } mainFlat := productRepo.makeProductFlat(mainPorduct.ID) if err := tx.Create(&mainFlat).Error; err != nil { tx.Rollback() + log.Println(err,"er5") return nil, err } @@ -304,9 +447,17 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er } savePoint := "size" + strconv.Itoa(index) tx.SavePoint(savePoint) - sizeOPtion := gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue) + + var sizeOPtion gm.AttributeOption + + if variant.AttributeName == "Beden" { + sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue) + } else { + sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue) + } sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) + variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku) variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) @@ -330,19 +481,30 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er if len(sizeVariants) == 0 { tx.Rollback() - return nil, errors.New("siz variantlary yok bolsa main productam girayenok") + return nil, errors.New("size variantlary yok bolsa main productam girayenok") } else { calcPrice(sizeVariants, &mainFlat) err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error + + mainFlat.Variants = sizeVariants + if err != nil { tx.Rollback() + log.Println(err,"er6") return nil, err } } } + sProduct := importer.createSellerProduct(&mainFlat, product.Vendor) + + if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil { + tx.Rollback() + return nil, errSProduct + } + if err := tx.Commit().Error; err != nil { return nil, err } @@ -350,9 +512,43 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er return &mainPorduct, nil } +func (importer *Importer) createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct { + sellerID := importer.sellers[sellerURL].ID + + if sellerID == 0 { + sellerID = 1 + } + + sellerProduct := gm.MarketplaceProduct{ + MarketplaceSellerID: sellerID, + IsApproved: true, + Condition: "new", + Description: "scraped", + IsOwner: true, + ProductID: flat.ProductID, + } + + for _, variant := range flat.Variants { + sellerProduct.Variants = append(sellerProduct.Variants, gm.MarketplaceProduct{ + ProductID: variant.ProductID, + IsOwner: true, + IsApproved: true, + MarketplaceSellerID: sellerID, + Condition: "new", + }) + } + + return sellerProduct + +} + func calcPrice(variants []gm.ProductFlat, flat *gm.ProductFlat) { for _, variant := range variants { + if !variant.Status { + continue + } + if flat.MinPrice == 0 || flat.MinPrice > variant.MinPrice { flat.MinPrice = variant.MinPrice } @@ -394,3 +590,461 @@ func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption { importer.SexMutex.Unlock() return option } + +func (importer *Importer) updateVariant(product models.Product) (*gm.Product, error) { + + var flat gm.ProductFlat + err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error + if err != nil { + if errors.Is(err,gorm.ErrRecordNotFound) { + return importer.importVariant(product) + } + //todo not found bolsa create etmeli + return nil, err + } + + if flat.Product.Type == "configurable" { + + productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) + if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil { + return nil, err + } else { + productRepo.Brand = brand + } + + for _, variant := range *product.SizeVariants { + + found := false + for _, flatVariant := range flat.Variants { + + if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel { + + if !variant.Sellable { + importer.baza.Model(&flatVariant).Update("status", false) + } else { + + importer.updatePrice(variant.Price, &flatVariant) + + if !flatVariant.Status { + + importer.baza.Model(&flatVariant).Update("status", true) + } + } + found = true + break + } + } + + if variant.Sellable && !found { + // insert variant + var sizeOPtion gm.AttributeOption + + if variant.AttributeName == "Beden" { + sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue) + } else { + sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue) + } + + sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) + variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku) + + variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) + + if err := importer.baza.Omit("Categories.*").Create(&variantProduct).Error; err != nil { + log.Println("Variant Product Create Error: " + err.Error()) + }else{ + variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID) + + if err := importer.baza.Create(&variantFlat).Error; err != nil { + log.Println("Variant Flat Create Error: " + err.Error()) + + } + flat.Variants = append(flat.Variants, variantFlat) + } + + } + } + + calcPrice(flat.Variants, &flat) + importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat) + + } else { //simple + + importer.updatePrice(product.Price, &flat) + + } + return &flat.Product, nil +} + +func (importer *Importer) updatePrice(price models.Price, flat *gm.ProductFlat) { + if price.OriginalPrice.Value > price.DiscountedPrice.Value { + + importer.baza.Model(&flat).Updates(map[string]interface{}{ + "price": price.OriginalPrice.Value, + "special_price": price.DiscountedPrice.Value, + "min_price": price.DiscountedPrice.Value, + "max_price": price.OriginalPrice.Value, + }) + + importer.baza.Model(&gm.ProductAttributeValue{}). + Where("attribute_id = 11 and product_id = ?", flat.ProductID). + Update("float_value", price.OriginalPrice.Value) + + importer.baza.Model(&gm.ProductAttributeValue{}). + Where("attribute_id = 13 and product_id = ?", flat.ProductID). + Update("float_value", price.DiscountedPrice.Value) + } else { + + importer.baza.Model(&flat).Updates(map[string]interface{}{ + "price": price.DiscountedPrice.Value, + "special_price": nil, + "min_price": price.DiscountedPrice.Value, + "max_price": price.DiscountedPrice.Value, + }) + + importer.baza.Model(&gm.ProductAttributeValue{}). + Where("attribute_id = 11 and product_id = ?", flat.ProductID). + Update("float_value", price.DiscountedPrice.Value) + + importer.baza.Where("attribute_id = 13 and product_id = ?", flat.ProductID). + Delete(&gm.ProductAttributeValue{}) + } +} + +func (importer *Importer) UpdateOrCreate(product models.Product) (instance *Importer) { + + firstProduct, err := importer.importVariant(product) + var newProducts []gm.Product + if err != nil { + helper.Error(err) + + firstProduct, importer.Error = importer.updateVariant(product) + + if importer.Error != nil { + return importer + } + } else if &firstProduct != nil { + newProducts = append(newProducts, *firstProduct) + } + + if product.ColorVariants != nil && len(*product.ColorVariants) > 0 { + linkedProducts := []gm.Product{*firstProduct} + + for _, colorVariant := range *product.ColorVariants { + + var ( + variant *gm.Product + err error + ) + if !colorVariant.IsSellable { + + if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil { + log.Println(err) + } + continue + + } else { + + if variant, err = importer.importVariant(colorVariant); err != nil { + + if variant, importer.Error = importer.updateVariant(colorVariant); importer.Error != nil { + return importer + } + + linkedProducts = append(linkedProducts, *variant) + + } else { + newProducts = append(newProducts, *variant) + } + } + + } + + if len(newProducts) > 0 { + // relation + + var relation []gm.ProductRelation + for _, linkedProduct := range linkedProducts { + + for _, newProd := range newProducts { + relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID}) + relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID}) + } + } + + if err := importer.baza.Create(&relation).Error; err != nil { + log.Println(err) + } + } + + } + return importer +} + +func (importer *Importer) importParsedVariant(product models.Product)(gm.Product, error) { + + //todo search if exists + + productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) + productRepo.ImageType = "lcw" + if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil { + return gm.Product{}, err + } else { + productRepo.Brand = brand + } + mainPorduct := productRepo.makeProduct(importer) + + //BEGIN TRANSACTION + tx := importer.baza.Begin() + + if err := tx.Omit("Categories", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil { + tx.Rollback() + return gm.Product{}, err + + } + + mainFlat := productRepo.makeProductFlat(mainPorduct.ID) + + if err := tx.Create(&mainFlat).Error; err != nil { + tx.Rollback() + return gm.Product{}, err + } + + if productRepo.HasSizeVariants() { + var sizeVariants []gm.ProductFlat + for index, variant := range *product.SizeVariants { + if !variant.Sellable { + continue + } + savePoint := "size" + strconv.Itoa(index) + tx.SavePoint(savePoint) + + var sizeOPtion gm.AttributeOption + + if variant.AttributeName == "Beden" { + sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue) + } else { + sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue) + } + + sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) + log.Println(sku) + log.Println(variant) + + variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku) + + variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) + + if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil { + log.Println("Variant Product Create Error: " + err.Error()) + tx.RollbackTo(savePoint) + continue + } + + variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID, variantProduct.ID) + + if err := tx.Create(&variantFlat).Error; err != nil { + log.Println("Variant Flat Create Error: " + err.Error()) + tx.RollbackTo(savePoint) + continue + } + + sizeVariants = append(sizeVariants, variantFlat) + } + + if len(sizeVariants) == 0 { + tx.Rollback() + return gm.Product{}, errors.New("siz variantlary yok bolsa main productam girayenok") + } else { + calcPrice(sizeVariants, &mainFlat) + + err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error + + mainFlat.Variants = sizeVariants + + if err != nil { + tx.Rollback() + return gm.Product{}, err + } + } + + } + + log.Println(product.Vendor) + + sProduct := importer.createSellerProduct(&mainFlat, product.Vendor) + + if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil { + tx.Rollback() + return gm.Product{}, errSProduct + } + + if err := tx.Commit().Error; err != nil { + return gm.Product{}, err + } + + return mainPorduct, nil + +} + +func (importer *Importer) updateParsedVariant(product models.Product) (gm.Product, error){ + var flat gm.ProductFlat + err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error + if err != nil { + if errors.Is(err,gorm.ErrRecordNotFound) { + return importer.importParsedVariant(product) + } + //todo not found bolsa create etmeli + return gm.Product{}, err + } + + if flat.Product.Type == "configurable" { + + productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet)) + productRepo.ImageType = "lcw" + if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil { + return gm.Product{}, err + } else { + productRepo.Brand = brand + } + + for _, variant := range *product.SizeVariants { + + found := false + for _, flatVariant := range flat.Variants { + + if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel { + + if !variant.Sellable { + importer.baza.Model(&flatVariant).Update("status", false) + } else { + + importer.updatePrice(variant.Price, &flatVariant) + + if !flatVariant.Status { + + importer.baza.Model(&flatVariant).Update("status", true) + } + } + found = true + break + } + } + + if variant.Sellable && !found { + // insert variant + var sizeOPtion gm.AttributeOption + + if variant.AttributeName == "Beden" { + sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue) + } else { + sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue) + } + log.Println(variant) + sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber) + log.Println(sku) + + + variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku) + + variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID) + + if err := importer.baza.Omit("Categories").Create(&variantProduct).Error; err != nil { + log.Println("Variant Product Create Error: " + err.Error()) + } + + variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID) + + if err := importer.baza.Create(&variantFlat).Error; err != nil { + log.Println("Variant Flat Create Error: " + err.Error()) + + } + flat.Variants = append(flat.Variants, variantFlat) + } + } + + calcPrice(flat.Variants, &flat) + importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat) + + } else { //simple + + importer.updatePrice(product.Price, &flat) + + } + return flat.Product, nil +} + +func (importer *Importer) UpdateOrCreateLCW(product models.Product) (instance *Importer){ + var firstProduct gm.Product + result := importer.baza.First(&firstProduct,"sku=?",product.ProductNumber) + + if errors.Is(result.Error, gorm.ErrRecordNotFound){ + firstProduct, importer.Error = importer.importParsedVariant(product) + + }else{ + firstProduct, importer.Error = importer.updateParsedVariant(product) + + } + + var newProducts []gm.Product + + if importer.Error != nil { + return importer + }else if &firstProduct != nil { + newProducts = append(newProducts, firstProduct) + } + + if product.ColorVariants != nil && len(*product.ColorVariants) > 0 { + linkedProducts := []gm.Product{firstProduct} + + for _, colorVariant := range *product.ColorVariants { + + var ( + variant gm.Product + err error + ) + if !colorVariant.IsSellable { + + if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil { + log.Println(err) + } + continue + + } else { + + //todo check parsed import variant + if variant, err = importer.importParsedVariant(colorVariant); err != nil { + + if variant, importer.Error = importer.updateParsedVariant(colorVariant); importer.Error != nil { + return importer + } + + linkedProducts = append(linkedProducts, variant) + + } else { + newProducts = append(newProducts, variant) + } + } + + } + + if len(newProducts) > 0 { + // relation + + var relation []gm.ProductRelation + for _, linkedProduct := range linkedProducts { + + for _, newProd := range newProducts { + relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID}) + relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID}) + } + } + + if err := importer.baza.Create(&relation).Error; err != nil { + log.Println(err) + } + } + + } + return importer +} diff --git a/repositories/LCWRepository.go b/repositories/LCWRepository.go new file mode 100644 index 0000000..0644af3 --- /dev/null +++ b/repositories/LCWRepository.go @@ -0,0 +1,29 @@ +package repositories + +import ( + "db_service/models" + helper "db_service/pkg" +) + +type LinkParserLCW struct { + link string +} + +func NewLinkParserLCW(link string) LinkParserLCW { + return LinkParserLCW{link: link} +} + +func (l LinkParserLCW) ParseLink() (*models.Product, error) { + + helper.Info("link: ", l.link) + + lcwScraper := NewLCWScraper(l.link) + + product, err := lcwScraper.InitProductDetailParsing() + if err != nil { + helper.Error(err) + return nil, err + } + + return &product, nil +} \ No newline at end of file diff --git a/repositories/ParseRepository.go b/repositories/ParseRepository.go index 7cad1a7..1360dd6 100644 --- a/repositories/ParseRepository.go +++ b/repositories/ParseRepository.go @@ -5,6 +5,7 @@ import ( helper "db_service/pkg" "encoding/json" "errors" + "fmt" "net/http" "reflect" "regexp" @@ -76,8 +77,6 @@ func getProductIdFromShortLink(shortLink string) string { productId = getProductIdFromLink(url) - helper.Info("productId: ", productId) - return productId } @@ -117,6 +116,8 @@ func GetProductDetails(productId string) (models.TrendyolProductDetailModel, err if err != nil { helper.Error(err) + helper.Info(body) + return productDetailModel, err } @@ -226,9 +227,9 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int json["images"] = model.Images json["brand"] = model.Brand.Name json["cinsiyet"] = model.Gender.Name - json["description"] = model.Description + //json["description"] = model.Description json["descriptions"] = model.ContentDescriptions - json["short_description"] = model.Description + //json["short_description"] = model.Description // nested structure json["price"] = make(map[string]interface{}) json["price"].(map[string]interface{})["originalPrice"] = model.Price.OriginalPrice @@ -266,7 +267,7 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int var variants []models.Variant // if show variants, then it is configurable product. - if model.ShowVariants { + if model.ShowVariants && len(model.Variants) != 0 { for i := 0; i < len(model.Variants); i++ { variant := model.Variants[i] @@ -288,30 +289,79 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int variants = append(variants, variant) } } - } } + } + for i := 0; i < len(model.AllVariants); i++ { + singleVariant := model.AllVariants[i] + + // get the first variant for attribute info + fv := variants[0] + + variant := models.Variant{ + AttributeID: fv.AttributeID, // + AttributeName: fv.AttributeName, // Sample: "Beden" + AttributeType: fv.AttributeType, // Sample: "Size" + AttributeValue: singleVariant.Value, + Price: models.Price{ + ProfitMargin: 0, + DiscountedPrice: struct { + Text string "json:\"text\"" + Value float64 "json:\"value\"" + }{ + Text: fmt.Sprintf("%f", singleVariant.Price), + Value: singleVariant.Price, + }, + SellingPrice: struct { + Text string "json:\"text\"" + Value float64 "json:\"value\"" + }{ + Text: fmt.Sprintf("%f", singleVariant.Price), + Value: singleVariant.Price, + }, + OriginalPrice: struct { + Text string "json:\"text\"" + Value float64 "json:\"value\"" + }{ + Text: fmt.Sprintf("%f", singleVariant.Price), + Value: singleVariant.Price, + }, + Currency: singleVariant.Currency, + }, + ItemNumber: singleVariant.ItemNumber, + Sellable: singleVariant.InStock, + } + + exists := helper.IsVariantsAdded(variants, variant.ItemNumber) + + if !exists { + variants = append(variants, variant) + } } for i := 0; i < len(model.AlternativeVariants); i++ { alternativeVariant := model.AlternativeVariants[i] - stockType := reflect.TypeOf(alternativeVariant.Stock) - - if stockType == nil && len(variants) > 0 { + if len(variants) > 0 { // get the first variant for attribute info fv := variants[0] variant := models.Variant{ - AttributeID: fv.AttributeID, - AttributeName: fv.AttributeName, - AttributeType: fv.AttributeType, + AttributeID: fv.AttributeID, // + AttributeName: fv.AttributeName, // Sample: "Beden" + AttributeType: fv.AttributeType, // Sample: "Size" AttributeValue: alternativeVariant.AttributeValue, Price: alternativeVariant.Price, + ItemNumber: alternativeVariant.ItemNumber, + Sellable: alternativeVariant.Quantity > 0, + } + exists := helper.IsVariantsAdded(variants, variant.ItemNumber) + + if !exists { + variants = append(variants, variant) } - variants = append(variants, variant) } } diff --git a/repositories/ProductRepository.go b/repositories/ProductRepository.go index e9632e3..b182299 100644 --- a/repositories/ProductRepository.go +++ b/repositories/ProductRepository.go @@ -18,6 +18,7 @@ type ProductRepo struct { Error error ColorOption gm.AttributeOption SexOption gm.AttributeOption + ImageType string } func InitProductRepo(data *models.Product, color, sex gm.AttributeOption) *ProductRepo { @@ -40,6 +41,7 @@ func InitProductRepo(data *models.Product, color, sex gm.AttributeOption) *Produ Data: data, ColorOption: color, SexOption: sex, + ImageType: "cdn", } return instance @@ -87,7 +89,13 @@ func (pr *ProductRepo) makeProduct(imp *Importer) gm.Product { if pr.HasSizeVariants() { product.Type = "configurable" - product.SuperAttributes = []gm.Attribute{imp.AttributesMap["size"]} + if (*pr.Data.SizeVariants)[0].AttributeName == "Beden" { + product.SuperAttributes = []gm.Attribute{imp.AttributesMap["size"]} + } else { + product.SuperAttributes = []gm.Attribute{imp.AttributesMap["boyut"]} + product.AttributeFamilyID = 4 // todo fix hardcode + } + } else { product.Type = "simple" price := pr.Data.Price @@ -105,7 +113,7 @@ func (pr *ProductRepo) makeProduct(imp *Importer) gm.Product { } for _, element := range pr.Data.Images { - product.Images = append(product.Images, gm.ProductImage{Type: "cdn", Path: element}) + product.Images = append(product.Images, gm.ProductImage{Type: pr.ImageType, Path: element}) } return product @@ -156,6 +164,11 @@ func (pr *ProductRepo) makeProductFlat(productId uint) gm.ProductFlat { flat.BrandID = pr.Brand.ID } + if pr.Data.Cinsiyet != "" { + flat.Cinsiyet = int(pr.SexOption.ID) + flat.CinsiyetLabel = pr.Data.Cinsiyet + } + if !pr.HasSizeVariants() { flat.MinPrice = pr.Data.Price.DiscountedPrice.Value flat.MaxPrice = math.Max(pr.Data.Price.OriginalPrice.Value, pr.Data.Price.DiscountedPrice.Value) @@ -200,10 +213,18 @@ func (pr *ProductRepo) makeVariantFlat(variant models.Variant, SizID, parentID, FavoritesCount: uint(pr.Data.FavoriteCount), SizeLabel: variant.AttributeValue, Size: int(SizID), - MaxPrice: maxPRice, - MinPrice: variant.Price.DiscountedPrice.Value, - Price: maxPRice, - ProductID: productID, + MaxPrice: maxPRice, + MinPrice: variant.Price.DiscountedPrice.Value, + Price: maxPRice, + ProductID: productID, + } + + if variant.AttributeName == "Beden" { + flat.Size = int(SizID) + flat.SizeLabel = variant.AttributeValue + } else { + flat.Boyut = int(SizID) + flat.BoyutLabel = variant.AttributeValue } if flat.MaxPrice > flat.MinPrice { @@ -215,6 +236,15 @@ func (pr *ProductRepo) makeVariantFlat(variant models.Variant, SizID, parentID, flat.ColorLabel = pr.Data.Color } + if pr.Data.Brand != "" { + flat.BrandID = pr.Brand.ID + } + + if pr.Data.Cinsiyet != "" { + flat.Cinsiyet = int(pr.SexOption.ID) + flat.CinsiyetLabel = pr.Data.Cinsiyet + } + return flat } @@ -253,9 +283,14 @@ func (pr *ProductRepo) getVariantAttributes(AttributesMap map[string]gm.Attribut {AttributeID: AttributesMap["name"].ID, TextValue: pr.Data.Name, Channel: "default", Locale: "tm"}, {AttributeID: AttributesMap["weight"].ID, TextValue: pr.Data.Weight}, {AttributeID: AttributesMap["status"].ID, BooleanValue: true}, - {AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)}, + //{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)}, {AttributeID: AttributesMap["price"].ID, FloatValue: price}, } + if product.AttributeName == "Beden" { + attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)}) + } else { + attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["boyut"].ID, IntegerValue: int(SizID)}) + } if pr.Data.Color != "" { attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["color"].ID, IntegerValue: int(pr.ColorOption.ID)}) diff --git a/repositories/scraper_lcw.go b/repositories/scraper_lcw.go new file mode 100644 index 0000000..97087b5 --- /dev/null +++ b/repositories/scraper_lcw.go @@ -0,0 +1,301 @@ +package repositories + +import ( + "context" + "db_service/models" + helper "db_service/pkg" + "fmt" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/chromedp/cdproto/dom" + "github.com/chromedp/chromedp" +) + +const ( + LCW_URL = "https://www.lcwaikiki.com" +) + +type LCWScraper struct { + link string +} + +func NewLCWScraper(link string) LCWScraper { + return LCWScraper{link: link} +} + +func (s LCWScraper) InitProductDetailParsing() (models.Product, error) { + + product := models.Product{} + + // load page with js + html, err := loadPage(s.link) + + if err != nil { + return product, err + } + + // convert to goquery document + doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + + if err != nil { + fmt.Println(err) + return product, err + } + + product = parseDocument(doc, true) + product.Vendor = "lcw" + return product, nil +} + +// loadPage loads page using chromeDp +// ChromeDp waits until JS render complete. returns html string +func loadPage(url string) (string, error) { + var res string + + ctx, cancel := chromedp.NewContext(context.Background()) + defer cancel() + + err := chromedp.Run(ctx, + chromedp.Navigate(url), + chromedp.ActionFunc(func(ctx context.Context) error { + node, err := dom.GetDocument().Do(ctx) + if err != nil { + return err + } + res, err = dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx) + return err + }), + ) + + if err != nil { + return res, err + } + + return res, nil +} + +//parseDocument parses document using goQuery +func parseDocument(doc *goquery.Document, primaryModel bool) models.Product { + + product := &models.Product{ + ColorVariants: &([]models.Product{}), + SizeVariants: &([]models.Variant{}), + } + + doc.Find("meta").Each(func(i int, s *goquery.Selection) { + name, _ := s.Attr("name") + content, _ := s.Attr("content") + switch name { + case "ModelId": + product.ProductGroupID = content + case "OptionId": + product.Sku = "p-" + content + product.ProductNumber = "lcw-"+content + case "ProductCode": + product.ProductCode = content + case "ProductName": + product.Name = content + case "BrandName": + product.Brand = content + case "Gender": + product.Cinsiyet = content + case "description": + product.Description = content + case "Color": + product.Color = content + case "DiscountPrice_1": + product.Price.SellingPrice.Text = "" + product.Price.SellingPrice.Value = convertStrToFloat(content) + case "CashPrice_1": + product.Price.OriginalPrice.Value = convertStrToFloat(content) + product.Weight = "0" + // case "Size": + // fmt.Println("SIZE: ", content) + // case "SizeId": + // fmt.Println("SIZE_ID: ", content) + } + }) + + productLink, ok := doc.Find(".share-link").Attr("value") + if ok { + urlKey := strings.ReplaceAll(productLink, LCW_URL, " ") + product.URLKey = strings.TrimSpace(urlKey) + } + + setupPrices(product, doc) + setupSizeVariants(product, doc) + setupSizeImages(product, doc) + + // call color variants for only primary model + if primaryModel { + setupColorVariants(product, doc) + } + + setupProductProperties(product, doc) + + return *product +} + +func setupPrices(product *models.Product, doc *goquery.Document) { + + basketPrice := strings.Trim(doc.Find(".col-xs-12 .price-area > .campaign-discount-detail > .basket-discount").Text(), " \n") + basketPriceFloat := convertStrToFloat(basketPrice) + + // check basket price discount + if basketPriceFloat != 0.0 { + product.Price.DiscountedPrice.Value = basketPriceFloat + } +} + +func setupSizeVariants(product *models.Product, doc *goquery.Document) { + + optionSizes := doc.Find(".option-size").Children() + for i := range optionSizes.Nodes { + optionSizes.Eq(i).Each(func(ii int, selection *goquery.Selection) { + sizeValue, _ := selection.Attr("size") + dataStock, _ := selection.Attr("data-stock") + key, _ := selection.Attr("key") + + keyInt, err := strconv.Atoi(key) + if err != nil { + // ... handle error + // panic(err) + fmt.Println("error in generating keyInt for ItemNumber") + } + + if len(dataStock) != 0 { + stock, _ := strconv.Atoi(dataStock) + + if stock > 3 { + sizeVariant := models.Variant{ + AttributeName: "Beden", + AttributeValue: sizeValue, + Stock: dataStock, + Sellable: true, + ItemNumber: keyInt, + Price: models.Price{ + DiscountedPrice: models.PriceValue{ + Text: "", + Value: product.Price.DiscountedPrice.Value, + }, + SellingPrice: models.PriceValue{ + Text: "", + Value: product.Price.SellingPrice.Value, + }, + OriginalPrice: models.PriceValue{ + Text: "", + Value: product.Price.OriginalPrice.Value, + }, + }, + + } + + isItemAdded := helper.IsLCWSizeVariantsAdded(*product.SizeVariants, sizeVariant) + if !isItemAdded { + *product.SizeVariants = append(*product.SizeVariants, sizeVariant) + } + } + } + }) + } +} + +//setupColorVariants gets color variant products. +func setupColorVariants(product *models.Product, doc *goquery.Document) { + + colorVariants := make([]models.Product, 0) + + doc.Find(".color-option").Each(func(i int, s *goquery.Selection) { + optionLink, ok := s.Attr("href") + if ok && !strings.Contains(optionLink, "javascript:") { + url := LCW_URL + optionLink + htmlPage, err := loadPage(url) + if err != nil { + return + } + + newDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlPage)) + if err != nil { + fmt.Println("err: ", err) + return + } + + colorOption := parseDocument(newDoc, false) + colorVariants = append(colorVariants, colorOption) + } + }) + + product.ColorVariantCount = len(colorVariants) + *product.ColorVariants = append(*product.ColorVariants, colorVariants...) +} + +func setupSizeImages(product *models.Product, doc *goquery.Document) { + + // images := make([]string, 0) + + doc.Find("img").Each(func(i int, s *goquery.Selection) { + if image, ok := s.Attr("smallimages"); ok { + isAdded := helper.IsImageAdded(product.Images,image) + if !isAdded { + product.Images = append(product.Images, image) + } + } + }) +} + +func setupProductProperties(product *models.Product, doc *goquery.Document) { + // get collapseOne + collapseOne := doc.Find("#collapseOne") + + // get descriptions + collapseOne.Find("li").Each(func(i int, selection *goquery.Selection) { + desc := strings.TrimSpace(selection.Text()) + if len(desc) != 0 { + desc := models.DescriptionModel{ + Description: desc, + Bold: false, + } + // descriptions = append(descriptions, desc) + product.Descriptions = append(product.Descriptions, desc) + } + }) + + collapseOne.Find(".option-info").Each(func(i int, selection *goquery.Selection) { + selection.Find("p").Each(func(i int, s *goquery.Selection) { + text := strings.Trim(s.Text(), "") + if len(text) != 0 { + attr := strings.Split(text, ":") + + attrKey, attrValue := strings.Trim(attr[0], " "), strings.Trim(attr[1], " ") + + if len(attrKey) != 0 && len(attrValue) != 0 { + attribute := map[string]string{ + strings.ToLower(attrKey): attrValue, + } + product.Attributes = append(product.Attributes, attribute) + } + } + }) + }) +} + +func convertStrToFloat(price string) float64 { + // remove TL + priceStr := strings.ReplaceAll(price, " TL", "") + // remove .(dot) + priceStr = strings.ReplaceAll(priceStr, ".", "") + // replace ,(comma) with .(dot) + priceStr = strings.ReplaceAll(priceStr, ",", ".") + // remove whitespace + priceStr = strings.ReplaceAll(priceStr, " ", "") + + priceFloat, err := strconv.ParseFloat(priceStr, 64) + + if err != nil { + return 0 + } + + return priceFloat +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0a2dac3..8623c9f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,6 +1,81 @@ +# github.com/PuerkitoBio/goquery v1.8.0 +## explicit; go 1.13 +github.com/PuerkitoBio/goquery +# github.com/andybalholm/cascadia v1.3.1 +## explicit; go 1.16 +github.com/andybalholm/cascadia +# github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24 +## explicit; go 1.19 +github.com/chromedp/cdproto +github.com/chromedp/cdproto/accessibility +github.com/chromedp/cdproto/animation +github.com/chromedp/cdproto/audits +github.com/chromedp/cdproto/backgroundservice +github.com/chromedp/cdproto/browser +github.com/chromedp/cdproto/cachestorage +github.com/chromedp/cdproto/cast +github.com/chromedp/cdproto/cdp +github.com/chromedp/cdproto/css +github.com/chromedp/cdproto/database +github.com/chromedp/cdproto/debugger +github.com/chromedp/cdproto/deviceorientation +github.com/chromedp/cdproto/dom +github.com/chromedp/cdproto/domdebugger +github.com/chromedp/cdproto/domsnapshot +github.com/chromedp/cdproto/domstorage +github.com/chromedp/cdproto/emulation +github.com/chromedp/cdproto/eventbreakpoints +github.com/chromedp/cdproto/fetch +github.com/chromedp/cdproto/headlessexperimental +github.com/chromedp/cdproto/heapprofiler +github.com/chromedp/cdproto/indexeddb +github.com/chromedp/cdproto/input +github.com/chromedp/cdproto/inspector +github.com/chromedp/cdproto/io +github.com/chromedp/cdproto/layertree +github.com/chromedp/cdproto/log +github.com/chromedp/cdproto/media +github.com/chromedp/cdproto/memory +github.com/chromedp/cdproto/network +github.com/chromedp/cdproto/overlay +github.com/chromedp/cdproto/page +github.com/chromedp/cdproto/performance +github.com/chromedp/cdproto/performancetimeline +github.com/chromedp/cdproto/profiler +github.com/chromedp/cdproto/runtime +github.com/chromedp/cdproto/security +github.com/chromedp/cdproto/serviceworker +github.com/chromedp/cdproto/storage +github.com/chromedp/cdproto/systeminfo +github.com/chromedp/cdproto/target +github.com/chromedp/cdproto/tethering +github.com/chromedp/cdproto/tracing +github.com/chromedp/cdproto/webaudio +github.com/chromedp/cdproto/webauthn +# github.com/chromedp/chromedp v0.8.7 +## explicit; go 1.18 +github.com/chromedp/chromedp +github.com/chromedp/chromedp/device +github.com/chromedp/chromedp/kb +# github.com/chromedp/sysutil v1.0.0 +## explicit; go 1.15 +github.com/chromedp/sysutil # github.com/go-sql-driver/mysql v1.6.0 ## explicit; go 1.10 github.com/go-sql-driver/mysql +# github.com/gobwas/httphead v0.1.0 +## explicit; go 1.15 +github.com/gobwas/httphead +# github.com/gobwas/pool v0.2.1 +## explicit +github.com/gobwas/pool +github.com/gobwas/pool/internal/pmath +github.com/gobwas/pool/pbufio +github.com/gobwas/pool/pbytes +# github.com/gobwas/ws v1.1.0 +## explicit; go 1.15 +github.com/gobwas/ws +github.com/gobwas/ws/wsutil # github.com/gorilla/mux v1.8.0 ## explicit; go 1.12 github.com/gorilla/mux @@ -19,9 +94,25 @@ github.com/jinzhu/now # github.com/joho/godotenv v1.4.0 ## explicit; go 1.12 github.com/joho/godotenv +# github.com/josharian/intern v1.0.0 +## explicit; go 1.5 +github.com/josharian/intern # github.com/leesper/couchdb-golang v1.2.1 ## explicit github.com/leesper/couchdb-golang +# github.com/mailru/easyjson v0.7.7 +## explicit; go 1.12 +github.com/mailru/easyjson +github.com/mailru/easyjson/buffer +github.com/mailru/easyjson/jlexer +github.com/mailru/easyjson/jwriter +# golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 +## explicit; go 1.17 +golang.org/x/net/html +golang.org/x/net/html/atom +# golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec +## explicit; go 1.17 +golang.org/x/sys/unix # gorm.io/driver/mysql v1.3.5 ## explicit; go 1.14 gorm.io/driver/mysql