merdan
/
go_service
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'parse'

This commit is contained in:
merdan 2023-09-22 18:19:27 +05:00
parent 3aa44579ab 07cc8a80e0
commit 9b6a83d866
23 changed files with 1523 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.env.example
View File

@ -1,17 +1,12 @@
couch_db_source=http://admin:adminadmin@192.168.1.2:5984/
couch_db_source=
db_trendyol=trendyol_db_v2
db_ty_categories=ty_categories
remote_url=https://sarga.com.tm/api/
remote_url=
descendant_category_url=descendant-categories
db=test_baza
du=merdan
dp=qazwsx12
da=localhost
dp=3306
#database_url ="merdan:qazwsx12@tcp(127.0.0.1:3306)/test_baza?parseTime=true"
database_url ="orient:orient@tcp(192.168.1.2:3306)/bagisto_sarga?parseTime=true"
couchdb_url = "http://admin:adminadmin@192.168.1.2:5984/"
scout_flash = "http://sarga.com.tm/scrap/scoute-flush"
database_url=
couchdb_url=
scout_flash=
port=

33
controllers/ImportController.go
View File

@ -1065,36 +1065,3 @@ func UpdateProduct(product models.Product, db *gorm.DB, productFlat gm.ProductFl
return errFlat
}
//func productAttributesAndFlat(data *models.Product) ([]gm.ProductAttributeValue,gm.ProductFlat){
//
// var description string
//
// for _, desc := range data.Descriptions {
// description += "<p>" + desc.Description + "</p>"
// }
//
// weight, _ := strconv.ParseFloat(data.Weight, 64)
//
// flat := gm.ProductFlat{
// Status: true,
// VisibleIndividually: true,
// Name: data.Name,
// Sku: data.ProductGroupID,
// ProductNumber: data.ProductNumber,
// Description: description,
// UrlKey: data.ProductGroupID,
// Weight: weight,
// FavoritesCount: uint(data.FavoriteCount),
// }
// return []gm.ProductAttributeValue{
// {AttributeID: AttributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount},
// {AttributeID: AttributesMap["source"].ID, TextValue: data.URLKey},
// {AttributeID: AttributesMap["product_number"].ID, TextValue: data.ProductNumber},
// {AttributeID: AttributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"},
// {AttributeID: AttributesMap["weight"].ID, TextValue: data.Weight},
// {AttributeID: AttributesMap["status"].ID, BooleanValue: true},
// {AttributeID: AttributesMap["visible_individually"].ID, BooleanValue: true},
// {AttributeID: AttributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
// },flat
//}

1
controllers/ImportV2.go
View File

@ -46,6 +46,7 @@ func StartProductImport(w http.ResponseWriter, _ *http.Request) {
}
importRepo.ImportWGroup.Wait()
//todo delete galan wishlist
//scout index flush
if err = gm.Flush(); err != nil {

142
controllers/ParseController.go
View File

@ -1,29 +1,82 @@
package controller
import (
gm "db_service/gorm_models"
"db_service/models"
helper "db_service/pkg"
"db_service/repositories"
"encoding/json"
"errors"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"fmt"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
)
func ParseLink(w http.ResponseWriter, route *http.Request) {
start := time.Now()
link := route.URL.Query().Get("url")
helper.Info("link: ", link)
if strings.Contains(link, "lcwaikiki.com"){
ParseLinkLCW(link,w)
}else {
ParseLinkTY(link,w)
}
}
func ParseLinkLCW(link string,w http.ResponseWriter) {
start := time.Now()
fmt.Println(link)
linkParser := repositories.NewLCWScraper(link)
product, err := linkParser.InitProductDetailParsing()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]string{
"msg": err.Error(),
})
fmt.Println(err)
return
}
/////////////////////////////
importRepo, err := repositories.ParseImporterInstance()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
fmt.Println(err)
return
}
//wait until initialization data is loaded to memory
importRepo.ImportWGroup.Wait()
log.Println(product);
if err = importRepo.UpdateOrCreateLCW(product).Error; err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
fmt.Println(err)
return
}
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": product.ProductNumber,
})
elapsed := time.Since(start)
log.Printf("end parse took %s", elapsed)
return
}
func ParseLinkTY(link string,w http.ResponseWriter) {
start := time.Now()
linkParser := repositories.NewLinkParser(link)
//todo
product, err := linkParser.ParseLink()
if err != nil {
@ -31,70 +84,75 @@ func ParseLink(w http.ResponseWriter, route *http.Request) {
json.NewEncoder(w).Encode(map[string]string{
"msg": err.Error(),
})
fmt.Println(err)
return
}
elapsed := time.Since(start)
log.Printf("link parsed at %s", elapsed)
jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID)
if err != nil {
helper.Error(err)
return
}
elapsed = time.Since(start)
log.Printf("product gat at %s", elapsed)
baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{})
if err != nil {
helper.Error(err)
return
}
var productFlat gm.ProductFlat
err = baza.Preload("Variants").
//Preload("Product").
//Preload("Proudct.AttributeValues","attribute_id in(11,13)").
First(&productFlat, "sku = ?", jsonProduct.ProductGroupID).Error
if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
helper.Error(err)
if len(AttributesMap) == 0 {
var attributes, err = gm.GetAttributes(baza)
/////////////////////////////
importRepo, err := repositories.ImporterInstance()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//wait until initialization data is loaded to memory
importRepo.ImportWGroup.Wait()
for _, atrattribute := range attributes {
AttributesMap[atrattribute.Code] = atrattribute
if err = importRepo.UpdateOrCreate(jsonProduct).Error; err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
err = ImportProduct(jsonProduct, baza)
} else if err == nil {
err = UpdateProduct(jsonProduct, baza, productFlat)
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": strconv.Itoa(product.ProductGroupID),
"productGroupId": strconv.Itoa(product.ID),
})
elapsed := time.Since(start)
log.Printf("end parse took %s", elapsed)
elapsed = time.Since(start)
log.Printf("imported at %s", elapsed)
return
/////////////////////////////////////////
}
func ImportLink(w http.ResponseWriter, route *http.Request) {
var product models.Product
err := json.NewDecoder(route.Body).Decode(&product)
if err != nil {
helper.Error(err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
/////////////////////////////
importRepo, err := repositories.ImporterInstance()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//wait until initialization data is loaded to memory
importRepo.ImportWGroup.Wait()
if err = importRepo.UpdateOrCreate(product).Error; err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
elapsed := time.Since(start)
log.Printf("end parse took %s", elapsed)
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"msg": "Link parsed successfully",
"productGroupId": strconv.Itoa(product.ProductGroupID),
"productGroupId": product.ProductGroupID,
})
}

12
go.mod
View File

@ -3,6 +3,9 @@ module db_service
go 1.19
require (
github.com/PuerkitoBio/goquery v1.8.0
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24
github.com/chromedp/chromedp v0.8.7
github.com/gorilla/mux v1.8.0
github.com/gosimple/slug v1.12.0
github.com/joho/godotenv v1.4.0
@ -12,8 +15,17 @@ require (
)
require (
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.1.0 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
)

32
go.sum
View File

@ -1,5 +1,21 @@
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24 h1:IKBU366KowxMcDpu9cgYEaXsiO0eBfPosTS0Krqfb8M=
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/chromedp v0.8.7 h1:dYOYc5ynTBzwSLOi+1IfgHwPr8r2BqV48l/RC+3OuJ0=
github.com/chromedp/chromedp v0.8.7/go.mod h1:iL+ywnwk3eG3EVXV1ackXBMNzdEh3Ye/KHvQkq1KRKU=
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gosimple/slug v1.12.0 h1:xzuhj7G7cGtd34NXnW/yF0l+AGNfWqwgh/IXgFy7dnc=
@ -13,8 +29,24 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc=
github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=

11
gorm_models/brand.go
View File

@ -26,14 +26,21 @@ func FindOrCreateBrand(db *gorm.DB, brand string, categories []Category) (Brand,
code := slug.Make(brand)
err := db.Omit("Categories").FirstOrCreate(&brandObject, Brand{Name: brand, Code: code, Categories: categories}).Error
// err := db.Model(&Brand{}).Find(&brandObject).Error
err := db.FirstOrCreate(&brandObject, Brand{Name: brand, Code: code}).Error
//err := db.Model(Brand{Name: brand, Code: code}).First(&brandObject).Error
//log.Println(Brand{Name: brand, Code: code})
if err != nil {
//err := db.Omit("Categories.*").FirstOrCreate(&brandObject, Brand{Name: brand, Code: code, Categories: categories}).Error
log.Println("ERR0000000000000001" + err.Error())
return brandObject, err
}
db.Model(&brandObject).Association("Categories").Append(categories)
return brandObject, nil
}
// var brandObject Brand

29
gorm_models/family.go
View File

@ -1,6 +1,7 @@
package gorm_models
import (
"errors"
"log"
"gorm.io/gorm"
@ -11,6 +12,14 @@ type AttributeOption struct {
AttributeID uint
AdminName string
SortOrder int `sql:"DEFAULT:NULL" gorm:"default:null"`
Translations []AttributeOptionTranslation
}
type AttributeOptionTranslation struct {
ID uint `gorm:"primaryKey"`
AttributeOptionID uint
Locale string
Label string
}
type Attribute struct {
@ -61,11 +70,27 @@ func GetAttributes(db *gorm.DB) ([]Attribute, error) {
func GetAttributeOption(db *gorm.DB, attrbuteID uint, value string) AttributeOption {
var option AttributeOption
err := db.FirstOrCreate(&option, AttributeOption{AttributeID: attrbuteID, AdminName: value}).Error
if err != nil {
err := db.Where("attribute_id=? and admin_name=?", attrbuteID, value).First(&option).Error
if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
option = AttributeOption{
AttributeID: attrbuteID,
AdminName: value,
SortOrder: 1000,
Translations: []AttributeOptionTranslation{
{Locale: "tm", Label: value},
{Locale: "ru", Label: value},
{Locale: "tr", Label: value},
},
}
if err := db.Create(&option).Error; err != nil {
log.Println(err.Error())
}
}
return option
}

43
gorm_models/product.go
View File

@ -4,6 +4,7 @@ import (
"context"
helper "db_service/pkg"
"gorm.io/gorm"
"log"
"os"
"time"
)
@ -38,7 +39,6 @@ type ProductFlat struct {
Name string `sql:"DEFAULT:''" gorm:"default:''"`
Weight float64 `sql:"DEFAULT:NULL" gorm:"type:decimal(12,4);default:null"`
Status bool `sql:"DEFAULT:NULL" gorm:"default:null"`
//Source string `sql:"DEFAULT:NULL" gorm:"default:null"`
VisibleIndividually bool `sql:"DEFAULT:NULL" gorm:"default:null"`
Price float64 `gorm:"type:decimal(12,4)"`
MinPrice float64 `gorm:"type:decimal(12,4)"`
@ -60,10 +60,14 @@ type ProductFlat struct {
ColorLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
Size int `sql:"DEFAULT:NULL" gorm:"default:null"`
SizeLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
Boyut int `sql:"DEFAULT:NULL" gorm:"default:null"`
BoyutLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
MetaTitle string `sql:"DEFAULT:NULL" gorm:"default:null"`
MetaKeywords string `sql:"DEFAULT:NULL" gorm:"default:null"`
BrandID uint `sql:"DEFAULT:NULL" gorm:"default:null"`
Brand Brand
Cinsiyet int `sql:"DEFAULT:NULL" gorm:"default:null"`
CinsiyetLabel string `sql:"DEFAULT:NULL" gorm:"default:null"`
}
type ProductImage struct {
@ -104,10 +108,43 @@ func DeleteProducts(db *gorm.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
//qb := "DELETE FROM products WHERE id NOT IN (select product_id as id from wishlist) AND id NOT IN (select product_id as id from order_items) AND id NOT IN (select parent_id as idfrom order_items);"
qb := "DELETE p FROM products p LEFT JOIN order_items oi ON p.id = oi.product_id LEFT JOIN order_items op ON p.id = op.parent_id WHERE oi.id IS NULL AND op.id IS NULL;"
return db.WithContext(ctx).Exec(qb).Error
qb := "DELETE p FROM products p " +
"LEFT JOIN order_items oi ON p.id = oi.product_id " +
"LEFT JOIN order_items op ON p.id = op.parent_id " +
"LEFT JOIN wishlist wp ON p.id = wp.product_id " +
"LEFT JOIN wishlist wl ON wl.product_id = p.parent_id " +
"JOIN marketplace_products mp ON p.id = mp.product_id "+
"WHERE oi.id IS NULL AND op.id IS NULL AND wp.id IS NULL AND wl.id IS NULL AND mp.marketplace_seller_id=1;"
db.WithContext(ctx).Exec(qb)
//db.WithContext(ctx).Exec("UPDATE product_flat set sku=concat(id,\"-ordered\"), status=0 where status=1 AND product_id IN (SELECT product_id from marketplace_products where marketplace_seller_id=1)" )
//db.WithContext(ctx).Exec("UPDATE products set sku=concat(id,\"-ordered\") WHERE id IN (SELECT product_id from marketplace_products where marketplace_seller_id=1)")
return db.Error
}
func GetWishlistProducts(db *gorm.DB) ([]Product, error){
var products [] Product
err := db.Joins("JOIN wishlist wp ON products.id = wp.product_id").Joins("JOIN marketplace_products mp ON products.id = mp.product_id where marketplace_seller_id=1").Find(&products).Error
if err != nil {
log.Println(err.Error())
return nil, err
}
return products, nil
//qb := "SELECT p.id,p.sku FROM products p " +
// "JOIN wishlist wp ON p.id = wp.product_id " +
// "JOIN marketplace_products mp ON p.id = mp.product_id where marketplace_seller_id=1);"
}
//func DisableProducts (db *gorm.DB) error {
// ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
// defer cancel()
// db.WithContext(ctx).Exec("UPDATE product_flat set sku=concat(sku,\"-ordered\"), status=0")
// db.WithContext(ctx).Exec("UPDATE products set sku=concat(sku,\"-ordered\")")
// return db.Error
//}
func Flush() error {
_, err := helper.SendRequest("GET", os.Getenv("scout_flash"), nil, "")

1
gorm_models/vendor.go
View File

@ -11,6 +11,7 @@ type MarketplaceProduct struct {
CreatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
UpdatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
ProductID uint
//Product Product
ParentID *uint `sql:"DEFAULT:NULL" gorm:"default:null"`
Condition string `sql:"DEFAULT:NULL" gorm:"default:null"`
Price float64

11
gorm_models/wishlist.go Normal file
View File

@ -0,0 +1,11 @@
package gorm_models
import "time"
type Wishlist struct {
ID uint `gorm:"primaryKey"`
CreatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
UpdatedAt time.Time `sql:"DEFAULT:NULL" gorm:"default:null"`
ProductID uint
//Product Product
}

1
main.go
View File

@ -49,6 +49,7 @@ func main() {
//route.HandleFunc("/init-importer", controller.StartImport)
route.HandleFunc("/init-importer", controller.StartProductImport)
route.HandleFunc("/parse-link", controller.ParseLink)
route.HandleFunc("/import-link", controller.ImportLink).Methods("POST", "GET")
err := http.ListenAndServe(os.Getenv("port"), route)
if err != nil {

17
models/lcw_scrap_data_model.go Normal file
View File

@ -0,0 +1,17 @@
package models
type LCWScrapDataModel struct {
Id string
Code string
Name string
OldPrice string
Price string
SizeVariants []SizeVariants
ColorOptions []LCWScrapDataModel
}
type DescriptionModel struct {
Description string `json:"description"`
Bold bool `json:"bold"`
}

22
models/product.go
View File

@ -33,21 +33,17 @@ type Product struct {
type Price struct {
ProfitMargin int `json:"profitMargin"`
DiscountedPrice struct {
Text string `json:"text"`
Value float64 `json:"value"`
} `json:"discountedPrice"`
SellingPrice struct {
Text string `json:"text"`
Value float64 `json:"value"`
} `json:"sellingPrice"`
OriginalPrice struct {
Text string `json:"text"`
Value float64 `json:"value"`
} `json:"originalPrice"`
DiscountedPrice PriceValue `json:"discountedPrice"`
SellingPrice PriceValue `json:"sellingPrice"`
OriginalPrice PriceValue `json:"originalPrice"`
Currency string `json:"currency"`
}
type PriceValue struct {
Text string `json:"text"`
Value float64 `json:"value"`
}
type Variant struct {
AttributeID int `json:"attributeId"`
AttributeName string `json:"attributeName"`
@ -163,7 +159,7 @@ type TrendyolProductDetailModel struct {
ProductCode string `json:"productCode"`
Name string `json:"name"`
NameWithProductCode string `json:"nameWithProductCode"`
Description string `json:"description"`
//Description string `json:"description"`
ContentDescriptions []struct {
Description string `json:"description"`
Bold bool `json:"bold"`

7
models/size_variant_model.go Normal file
View File

@ -0,0 +1,7 @@
package models
type SizeVariants struct {
Price float64 `json:"price"`
Size string `json:"size"`
Stock int `json:"stock"`
}

26
pkg/helper.go
View File

@ -2,6 +2,7 @@ package helper
import (
"context"
"db_service/models"
"log"
"net"
"net/http"
@ -84,3 +85,28 @@ func GetCategoryWeight(slug string) string {
return doc["weight"].(string)
}
func IsLCWSizeVariantsAdded(items []models.Variant, keyItem models.Variant) bool {
for _, option := range items {
if option.ItemNumber == keyItem.ItemNumber {
return true
}
}
return false
}
func IsImageAdded(images []string, image string) bool {
for _, img := range images {
if img == image {
return true
}
}
return false
}
func IsVariantsAdded(variants []models.Variant, itemNumber int) bool {
for _, variant := range variants {
if variant.ItemNumber == itemNumber {
return true
}
}
return false
}

4
pkg/request.go
View File

@ -12,8 +12,8 @@ import (
func SendRequest(method string, endpoint string, values []byte, authKey string) ([]byte, error) {
const ConnectMaxWaitTime = 30 * time.Second
const RequestMaxWaitTime = 60 * time.Second
const ConnectMaxWaitTime = time.Minute
const RequestMaxWaitTime = 5 * time.Minute
client := http.Client{
Transport: &http.Transport{

662
repositories/ImportRepository.go
View File

@ -22,6 +22,7 @@ type Importer struct {
baza *gorm.DB
families []gm.AttributeFamily
sellers map[string]gm.MarketplaceSeller
//wishlist map[string]gm.Product
AttributesMap map[string]gm.Attribute
Error error
ImportWGroup sync.WaitGroup
@ -30,7 +31,10 @@ type Importer struct {
ColorMutex sync.Mutex
SexMutex sync.Mutex
}
type GormErr struct {
Number int `json:"Number"`
Message string `json:"Message"`
}
func ImporterInstance() (instance *Importer, err error) {
db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})
@ -48,6 +52,7 @@ func ImporterInstance() (instance *Importer, err error) {
go func(db *gorm.DB) {
defer instance.ImportWGroup.Done()
instance.mainCategories, instance.Error = gm.GetMainCategories(db)
}(db)
//load families to memory
@ -113,6 +118,105 @@ func ImporterInstance() (instance *Importer, err error) {
}
}()
//load wishlist to memory
//go func(){
// defer instance.ImportWGroup.Done()
//
// var wishlist, err = gm.GetWishlistProducts(db)
//
// if err != nil {
// instance.Error = err
// return
// }
//
// instance.wishlist = make(map[string]gm.Product, len(wishlist))
//
// for _, product := range wishlist {
// instance.wishlist[product.Sku] = product
// }
//}()
if instance.Error != nil {
log.Println(instance.Error)
return nil, instance.Error
}
return instance, nil
}
func ParseImporterInstance() (instance *Importer, err error) {
db, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{SkipDefaultTransaction: true})
if err != nil {
log.Println(err)
return nil, err
}
instance = &Importer{baza: db}
instance.ImportWGroup.Add(3)
//load families to memory
go func() {
defer instance.ImportWGroup.Done()
instance.families, instance.Error = gm.GetFamilies(db)
}()
//load attributes to memory
go func() {
defer instance.ImportWGroup.Done()
if attributes, err := gm.GetAttributes(db); err != nil {
instance.Error = err
return
} else {
instance.AttributesMap = make(map[string]gm.Attribute, len(attributes))
for _, attribute := range attributes {
instance.AttributesMap[attribute.Code] = attribute
}
}
if colorOptions, err := gm.GetAttrOptions(db, instance.AttributesMap["color"].ID); err != nil {
instance.Error = err
return
} else {
instance.ColorOptions = make(map[string]gm.AttributeOption, len(colorOptions))
for _, option := range colorOptions {
instance.ColorOptions[option.AdminName] = option
}
}
if sexOPtions, err := gm.GetAttrOptions(db, instance.AttributesMap["cinsiyet"].ID); err != nil {
instance.Error = err
return
} else {
instance.SexOptions = make(map[string]gm.AttributeOption, len(sexOPtions))
for _, option := range sexOPtions {
instance.SexOptions[option.AdminName] = option
}
}
}()
//load sellers to memory
go func() {
defer instance.ImportWGroup.Done()
var vendors, err = gm.GetSellers(db)
if err != nil {
instance.Error = err
return
}
//init sellers map
instance.sellers = make(map[string]gm.MarketplaceSeller, len(vendors))
for _, vendor := range vendors {
instance.sellers[vendor.Url] = vendor
}
}()
if instance.Error != nil {
log.Println(instance.Error)
return nil, instance.Error
@ -237,7 +341,7 @@ func (importer *Importer) ImportProduct(product models.Product) (instance *Impor
}
if len(linkedProducts) > 1 {
//todo link products
var relation []gm.ProductRelation
for index, variant := range linkedProducts {
//spoint := "color" + strconv.Itoa(index)
@ -265,6 +369,13 @@ func (importer *Importer) ImportProduct(product models.Product) (instance *Impor
}
func (importer *Importer) importVariant(product models.Product) (*gm.Product, error) {
// check if wishlisted then update if.
//if _, ok := importer.wishlist[product.ProductNumber]; ok {
// delete(importer.wishlist,product.ProductNumber)
// return importer.updateVariant(product)
//}
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
if categories, err := gm.GetCatKeywords(importer.baza, product.Categories); err != nil {
@ -285,14 +396,46 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
tx := importer.baza.Begin()
if err := tx.Omit("Categories.*", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil {
//todo update categories
byteErr, _ := json.Marshal(err)
var newError GormErr
if err1:= json.Unmarshal((byteErr), &newError);err1!=nil{
tx.Rollback()
log.Println(err1,"err2")
return nil, err1
}
if newError.Number == 1062{
var barProduct gm.Product
if err2 := tx.First(&barProduct,"sku = ?",mainPorduct.Sku).Error;err2 !=nil {
tx.Rollback()
log.Println(err2,"err3")
return nil, err2
}
if err3:= tx.Model(&barProduct).Association("Categories").Append(mainPorduct.Categories); err3!=nil{
tx.Rollback()
log.Println(err3,"err4")
return nil, err3
}
if err4 := tx.Commit().Error; err4 ==nil{
return importer.updateVariant(product)
}
}else{
tx.Rollback()
log.Println(err,"er1")
return nil, err
}
}
mainFlat := productRepo.makeProductFlat(mainPorduct.ID)
if err := tx.Create(&mainFlat).Error; err != nil {
tx.Rollback()
log.Println(err,"er5")
return nil, err
}
@ -304,9 +447,17 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
}
savePoint := "size" + strconv.Itoa(index)
tx.SavePoint(savePoint)
sizeOPtion := gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
@ -330,19 +481,30 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
if len(sizeVariants) == 0 {
tx.Rollback()
return nil, errors.New("siz variantlary yok bolsa main productam girayenok")
return nil, errors.New("size variantlary yok bolsa main productam girayenok")
} else {
calcPrice(sizeVariants, &mainFlat)
err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error
mainFlat.Variants = sizeVariants
if err != nil {
tx.Rollback()
log.Println(err,"er6")
return nil, err
}
}
}
sProduct := importer.createSellerProduct(&mainFlat, product.Vendor)
if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil {
tx.Rollback()
return nil, errSProduct
}
if err := tx.Commit().Error; err != nil {
return nil, err
}
@ -350,9 +512,43 @@ func (importer *Importer) importVariant(product models.Product) (*gm.Product, er
return &mainPorduct, nil
}
func (importer *Importer) createSellerProduct(flat *gm.ProductFlat, sellerURL string) gm.MarketplaceProduct {
sellerID := importer.sellers[sellerURL].ID
if sellerID == 0 {
sellerID = 1
}
sellerProduct := gm.MarketplaceProduct{
MarketplaceSellerID: sellerID,
IsApproved: true,
Condition: "new",
Description: "scraped",
IsOwner: true,
ProductID: flat.ProductID,
}
for _, variant := range flat.Variants {
sellerProduct.Variants = append(sellerProduct.Variants, gm.MarketplaceProduct{
ProductID: variant.ProductID,
IsOwner: true,
IsApproved: true,
MarketplaceSellerID: sellerID,
Condition: "new",
})
}
return sellerProduct
}
func calcPrice(variants []gm.ProductFlat, flat *gm.ProductFlat) {
for _, variant := range variants {
if !variant.Status {
continue
}
if flat.MinPrice == 0 || flat.MinPrice > variant.MinPrice {
flat.MinPrice = variant.MinPrice
}
@ -394,3 +590,461 @@ func (importer *Importer) GetSexOption(optionName string) gm.AttributeOption {
importer.SexMutex.Unlock()
return option
}
func (importer *Importer) updateVariant(product models.Product) (*gm.Product, error) {
var flat gm.ProductFlat
err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error
if err != nil {
if errors.Is(err,gorm.ErrRecordNotFound) {
return importer.importVariant(product)
}
//todo not found bolsa create etmeli
return nil, err
}
if flat.Product.Type == "configurable" {
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
return nil, err
} else {
productRepo.Brand = brand
}
for _, variant := range *product.SizeVariants {
found := false
for _, flatVariant := range flat.Variants {
if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel {
if !variant.Sellable {
importer.baza.Model(&flatVariant).Update("status", false)
} else {
importer.updatePrice(variant.Price, &flatVariant)
if !flatVariant.Status {
importer.baza.Model(&flatVariant).Update("status", true)
}
}
found = true
break
}
}
if variant.Sellable && !found {
// insert variant
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
if err := importer.baza.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
log.Println("Variant Product Create Error: " + err.Error())
}else{
variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID)
if err := importer.baza.Create(&variantFlat).Error; err != nil {
log.Println("Variant Flat Create Error: " + err.Error())
}
flat.Variants = append(flat.Variants, variantFlat)
}
}
}
calcPrice(flat.Variants, &flat)
importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat)
} else { //simple
importer.updatePrice(product.Price, &flat)
}
return &flat.Product, nil
}
func (importer *Importer) updatePrice(price models.Price, flat *gm.ProductFlat) {
if price.OriginalPrice.Value > price.DiscountedPrice.Value {
importer.baza.Model(&flat).Updates(map[string]interface{}{
"price": price.OriginalPrice.Value,
"special_price": price.DiscountedPrice.Value,
"min_price": price.DiscountedPrice.Value,
"max_price": price.OriginalPrice.Value,
})
importer.baza.Model(&gm.ProductAttributeValue{}).
Where("attribute_id = 11 and product_id = ?", flat.ProductID).
Update("float_value", price.OriginalPrice.Value)
importer.baza.Model(&gm.ProductAttributeValue{}).
Where("attribute_id = 13 and product_id = ?", flat.ProductID).
Update("float_value", price.DiscountedPrice.Value)
} else {
importer.baza.Model(&flat).Updates(map[string]interface{}{
"price": price.DiscountedPrice.Value,
"special_price": nil,
"min_price": price.DiscountedPrice.Value,
"max_price": price.DiscountedPrice.Value,
})
importer.baza.Model(&gm.ProductAttributeValue{}).
Where("attribute_id = 11 and product_id = ?", flat.ProductID).
Update("float_value", price.DiscountedPrice.Value)
importer.baza.Where("attribute_id = 13 and product_id = ?", flat.ProductID).
Delete(&gm.ProductAttributeValue{})
}
}
func (importer *Importer) UpdateOrCreate(product models.Product) (instance *Importer) {
firstProduct, err := importer.importVariant(product)
var newProducts []gm.Product
if err != nil {
helper.Error(err)
firstProduct, importer.Error = importer.updateVariant(product)
if importer.Error != nil {
return importer
}
} else if &firstProduct != nil {
newProducts = append(newProducts, *firstProduct)
}
if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
linkedProducts := []gm.Product{*firstProduct}
for _, colorVariant := range *product.ColorVariants {
var (
variant *gm.Product
err error
)
if !colorVariant.IsSellable {
if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil {
log.Println(err)
}
continue
} else {
if variant, err = importer.importVariant(colorVariant); err != nil {
if variant, importer.Error = importer.updateVariant(colorVariant); importer.Error != nil {
return importer
}
linkedProducts = append(linkedProducts, *variant)
} else {
newProducts = append(newProducts, *variant)
}
}
}
if len(newProducts) > 0 {
// relation
var relation []gm.ProductRelation
for _, linkedProduct := range linkedProducts {
for _, newProd := range newProducts {
relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID})
relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID})
}
}
if err := importer.baza.Create(&relation).Error; err != nil {
log.Println(err)
}
}
}
return importer
}
func (importer *Importer) importParsedVariant(product models.Product)(gm.Product, error) {
//todo search if exists
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
productRepo.ImageType = "lcw"
if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
return gm.Product{}, err
} else {
productRepo.Brand = brand
}
mainPorduct := productRepo.makeProduct(importer)
//BEGIN TRANSACTION
tx := importer.baza.Begin()
if err := tx.Omit("Categories", "SuperAttributes.*", "ParentID").Create(&mainPorduct).Error; err != nil {
tx.Rollback()
return gm.Product{}, err
}
mainFlat := productRepo.makeProductFlat(mainPorduct.ID)
if err := tx.Create(&mainFlat).Error; err != nil {
tx.Rollback()
return gm.Product{}, err
}
if productRepo.HasSizeVariants() {
var sizeVariants []gm.ProductFlat
for index, variant := range *product.SizeVariants {
if !variant.Sellable {
continue
}
savePoint := "size" + strconv.Itoa(index)
tx.SavePoint(savePoint)
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(tx, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
log.Println(sku)
log.Println(variant)
variantProduct := productRepo.makeVariant(mainPorduct.ID, mainPorduct.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
if err := tx.Omit("Categories.*").Create(&variantProduct).Error; err != nil {
log.Println("Variant Product Create Error: " + err.Error())
tx.RollbackTo(savePoint)
continue
}
variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, mainFlat.ID, variantProduct.ID)
if err := tx.Create(&variantFlat).Error; err != nil {
log.Println("Variant Flat Create Error: " + err.Error())
tx.RollbackTo(savePoint)
continue
}
sizeVariants = append(sizeVariants, variantFlat)
}
if len(sizeVariants) == 0 {
tx.Rollback()
return gm.Product{}, errors.New("siz variantlary yok bolsa main productam girayenok")
} else {
calcPrice(sizeVariants, &mainFlat)
err := tx.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&mainFlat).Error
mainFlat.Variants = sizeVariants
if err != nil {
tx.Rollback()
return gm.Product{}, err
}
}
}
log.Println(product.Vendor)
sProduct := importer.createSellerProduct(&mainFlat, product.Vendor)
if errSProduct := tx.Create(&sProduct).Error; errSProduct != nil {
tx.Rollback()
return gm.Product{}, errSProduct
}
if err := tx.Commit().Error; err != nil {
return gm.Product{}, err
}
return mainPorduct, nil
}
func (importer *Importer) updateParsedVariant(product models.Product) (gm.Product, error){
var flat gm.ProductFlat
err := importer.baza.Preload("Product").Preload("Variants").First(&flat, "sku = ?", product.ProductNumber).Error
if err != nil {
if errors.Is(err,gorm.ErrRecordNotFound) {
return importer.importParsedVariant(product)
}
//todo not found bolsa create etmeli
return gm.Product{}, err
}
if flat.Product.Type == "configurable" {
productRepo := InitProductRepo(&product, importer.GetColorOption(product.Color), importer.GetSexOption(product.Cinsiyet))
productRepo.ImageType = "lcw"
if brand, err := gm.FindOrCreateBrand(importer.baza, product.Brand, productRepo.Categories); err != nil {
return gm.Product{}, err
} else {
productRepo.Brand = brand
}
for _, variant := range *product.SizeVariants {
found := false
for _, flatVariant := range flat.Variants {
if variant.AttributeValue == flatVariant.BoyutLabel || variant.AttributeValue == flatVariant.SizeLabel {
if !variant.Sellable {
importer.baza.Model(&flatVariant).Update("status", false)
} else {
importer.updatePrice(variant.Price, &flatVariant)
if !flatVariant.Status {
importer.baza.Model(&flatVariant).Update("status", true)
}
}
found = true
break
}
}
if variant.Sellable && !found {
// insert variant
var sizeOPtion gm.AttributeOption
if variant.AttributeName == "Beden" {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["size"].ID, variant.AttributeValue)
} else {
sizeOPtion = gm.GetAttributeOption(importer.baza, importer.AttributesMap["boyut"].ID, variant.AttributeValue)
}
log.Println(variant)
sku := fmt.Sprintf("%s-%d", product.ProductNumber, variant.ItemNumber)
log.Println(sku)
variantProduct := productRepo.makeVariant(flat.ProductID, flat.Product.AttributeFamilyID, sku)
variantProduct.AttributeValues = productRepo.getVariantAttributes(importer.AttributesMap, &variant, sizeOPtion.ID)
if err := importer.baza.Omit("Categories").Create(&variantProduct).Error; err != nil {
log.Println("Variant Product Create Error: " + err.Error())
}
variantFlat := productRepo.makeVariantFlat(variant, sizeOPtion.ID, flat.ID, variantProduct.ID)
if err := importer.baza.Create(&variantFlat).Error; err != nil {
log.Println("Variant Flat Create Error: " + err.Error())
}
flat.Variants = append(flat.Variants, variantFlat)
}
}
calcPrice(flat.Variants, &flat)
importer.baza.Omit("ParentID", "CreatedAt", "Variants", "SpecialPrice").Save(&flat)
} else { //simple
importer.updatePrice(product.Price, &flat)
}
return flat.Product, nil
}
func (importer *Importer) UpdateOrCreateLCW(product models.Product) (instance *Importer){
var firstProduct gm.Product
result := importer.baza.First(&firstProduct,"sku=?",product.ProductNumber)
if errors.Is(result.Error, gorm.ErrRecordNotFound){
firstProduct, importer.Error = importer.importParsedVariant(product)
}else{
firstProduct, importer.Error = importer.updateParsedVariant(product)
}
var newProducts []gm.Product
if importer.Error != nil {
return importer
}else if &firstProduct != nil {
newProducts = append(newProducts, firstProduct)
}
if product.ColorVariants != nil && len(*product.ColorVariants) > 0 {
linkedProducts := []gm.Product{firstProduct}
for _, colorVariant := range *product.ColorVariants {
var (
variant gm.Product
err error
)
if !colorVariant.IsSellable {
if err = importer.baza.Model(&gm.ProductFlat{}).Where("sku=?", colorVariant.ProductNumber).Update("status", false).Error; err != nil {
log.Println(err)
}
continue
} else {
//todo check parsed import variant
if variant, err = importer.importParsedVariant(colorVariant); err != nil {
if variant, importer.Error = importer.updateParsedVariant(colorVariant); importer.Error != nil {
return importer
}
linkedProducts = append(linkedProducts, variant)
} else {
newProducts = append(newProducts, variant)
}
}
}
if len(newProducts) > 0 {
// relation
var relation []gm.ProductRelation
for _, linkedProduct := range linkedProducts {
for _, newProd := range newProducts {
relation = append(relation, gm.ProductRelation{ParentID: linkedProduct.ID, ChildID: newProd.ID})
relation = append(relation, gm.ProductRelation{ParentID: newProd.ID, ChildID: linkedProduct.ID})
}
}
if err := importer.baza.Create(&relation).Error; err != nil {
log.Println(err)
}
}
}
return importer
}

29
repositories/LCWRepository.go Normal file
View File

@ -0,0 +1,29 @@
package repositories
import (
"db_service/models"
helper "db_service/pkg"
)
type LinkParserLCW struct {
link string
}
func NewLinkParserLCW(link string) LinkParserLCW {
return LinkParserLCW{link: link}
}
func (l LinkParserLCW) ParseLink() (*models.Product, error) {
helper.Info("link: ", l.link)
lcwScraper := NewLCWScraper(l.link)
product, err := lcwScraper.InitProductDetailParsing()
if err != nil {
helper.Error(err)
return nil, err
}
return &product, nil
}

84
repositories/ParseRepository.go
View File

@ -5,6 +5,7 @@ import (
helper "db_service/pkg"
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"regexp"
@ -76,8 +77,6 @@ func getProductIdFromShortLink(shortLink string) string {
productId = getProductIdFromLink(url)
helper.Info("productId: ", productId)
return productId
}
@ -117,6 +116,8 @@ func GetProductDetails(productId string) (models.TrendyolProductDetailModel, err
if err != nil {
helper.Error(err)
helper.Info(body)
return productDetailModel, err
}
@ -226,9 +227,9 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int
json["images"] = model.Images
json["brand"] = model.Brand.Name
json["cinsiyet"] = model.Gender.Name
json["description"] = model.Description
//json["description"] = model.Description
json["descriptions"] = model.ContentDescriptions
json["short_description"] = model.Description
//json["short_description"] = model.Description
// nested structure
json["price"] = make(map[string]interface{})
json["price"].(map[string]interface{})["originalPrice"] = model.Price.OriginalPrice
@ -266,7 +267,7 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int
var variants []models.Variant
// if show variants, then it is configurable product.
if model.ShowVariants {
if model.ShowVariants && len(model.Variants) != 0 {
for i := 0; i < len(model.Variants); i++ {
variant := model.Variants[i]
@ -288,32 +289,81 @@ func CreateJSONFromModel(model models.TrendyolProductDetailModel) map[string]int
variants = append(variants, variant)
}
}
}
}
}
}
for i := 0; i < len(model.AlternativeVariants); i++ {
alternativeVariant := model.AlternativeVariants[i]
stockType := reflect.TypeOf(alternativeVariant.Stock)
if stockType == nil && len(variants) > 0 {
for i := 0; i < len(model.AllVariants); i++ {
singleVariant := model.AllVariants[i]
// get the first variant for attribute info
fv := variants[0]
variant := models.Variant{
AttributeID: fv.AttributeID,
AttributeName: fv.AttributeName,
AttributeType: fv.AttributeType,
AttributeID: fv.AttributeID, //
AttributeName: fv.AttributeName, // Sample: "Beden"
AttributeType: fv.AttributeType, // Sample: "Size"
AttributeValue: singleVariant.Value,
Price: models.Price{
ProfitMargin: 0,
DiscountedPrice: struct {
Text string "json:\"text\""
Value float64 "json:\"value\""
}{
Text: fmt.Sprintf("%f", singleVariant.Price),
Value: singleVariant.Price,
},
SellingPrice: struct {
Text string "json:\"text\""
Value float64 "json:\"value\""
}{
Text: fmt.Sprintf("%f", singleVariant.Price),
Value: singleVariant.Price,
},
OriginalPrice: struct {
Text string "json:\"text\""
Value float64 "json:\"value\""
}{
Text: fmt.Sprintf("%f", singleVariant.Price),
Value: singleVariant.Price,
},
Currency: singleVariant.Currency,
},
ItemNumber: singleVariant.ItemNumber,
Sellable: singleVariant.InStock,
}
exists := helper.IsVariantsAdded(variants, variant.ItemNumber)
if !exists {
variants = append(variants, variant)
}
}
for i := 0; i < len(model.AlternativeVariants); i++ {
alternativeVariant := model.AlternativeVariants[i]
if len(variants) > 0 {
// get the first variant for attribute info
fv := variants[0]
variant := models.Variant{
AttributeID: fv.AttributeID, //
AttributeName: fv.AttributeName, // Sample: "Beden"
AttributeType: fv.AttributeType, // Sample: "Size"
AttributeValue: alternativeVariant.AttributeValue,
Price: alternativeVariant.Price,
ItemNumber: alternativeVariant.ItemNumber,
Sellable: alternativeVariant.Quantity > 0,
}
exists := helper.IsVariantsAdded(variants, variant.ItemNumber)
if !exists {
variants = append(variants, variant)
}
}
}
json["size_variants"] = variants
}

39
repositories/ProductRepository.go
View File

@ -18,6 +18,7 @@ type ProductRepo struct {
Error error
ColorOption gm.AttributeOption
SexOption gm.AttributeOption
ImageType string
}
func InitProductRepo(data *models.Product, color, sex gm.AttributeOption) *ProductRepo {
@ -40,6 +41,7 @@ func InitProductRepo(data *models.Product, color, sex gm.AttributeOption) *Produ
Data: data,
ColorOption: color,
SexOption: sex,
ImageType: "cdn",
}
return instance
@ -87,7 +89,13 @@ func (pr *ProductRepo) makeProduct(imp *Importer) gm.Product {
if pr.HasSizeVariants() {
product.Type = "configurable"
if (*pr.Data.SizeVariants)[0].AttributeName == "Beden" {
product.SuperAttributes = []gm.Attribute{imp.AttributesMap["size"]}
} else {
product.SuperAttributes = []gm.Attribute{imp.AttributesMap["boyut"]}
product.AttributeFamilyID = 4 // todo fix hardcode
}
} else {
product.Type = "simple"
price := pr.Data.Price
@ -105,7 +113,7 @@ func (pr *ProductRepo) makeProduct(imp *Importer) gm.Product {
}
for _, element := range pr.Data.Images {
product.Images = append(product.Images, gm.ProductImage{Type: "cdn", Path: element})
product.Images = append(product.Images, gm.ProductImage{Type: pr.ImageType, Path: element})
}
return product
@ -156,6 +164,11 @@ func (pr *ProductRepo) makeProductFlat(productId uint) gm.ProductFlat {
flat.BrandID = pr.Brand.ID
}
if pr.Data.Cinsiyet != "" {
flat.Cinsiyet = int(pr.SexOption.ID)
flat.CinsiyetLabel = pr.Data.Cinsiyet
}
if !pr.HasSizeVariants() {
flat.MinPrice = pr.Data.Price.DiscountedPrice.Value
flat.MaxPrice = math.Max(pr.Data.Price.OriginalPrice.Value, pr.Data.Price.DiscountedPrice.Value)
@ -206,6 +219,14 @@ func (pr *ProductRepo) makeVariantFlat(variant models.Variant, SizID, parentID,
ProductID: productID,
}
if variant.AttributeName == "Beden" {
flat.Size = int(SizID)
flat.SizeLabel = variant.AttributeValue
} else {
flat.Boyut = int(SizID)
flat.BoyutLabel = variant.AttributeValue
}
if flat.MaxPrice > flat.MinPrice {
flat.SpecialPrice = flat.MinPrice
}
@ -215,6 +236,15 @@ func (pr *ProductRepo) makeVariantFlat(variant models.Variant, SizID, parentID,
flat.ColorLabel = pr.Data.Color
}
if pr.Data.Brand != "" {
flat.BrandID = pr.Brand.ID
}
if pr.Data.Cinsiyet != "" {
flat.Cinsiyet = int(pr.SexOption.ID)
flat.CinsiyetLabel = pr.Data.Cinsiyet
}
return flat
}
@ -253,9 +283,14 @@ func (pr *ProductRepo) getVariantAttributes(AttributesMap map[string]gm.Attribut
{AttributeID: AttributesMap["name"].ID, TextValue: pr.Data.Name, Channel: "default", Locale: "tm"},
{AttributeID: AttributesMap["weight"].ID, TextValue: pr.Data.Weight},
{AttributeID: AttributesMap["status"].ID, BooleanValue: true},
{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)},
//{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)},
{AttributeID: AttributesMap["price"].ID, FloatValue: price},
}
if product.AttributeName == "Beden" {
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["size"].ID, IntegerValue: int(SizID)})
} else {
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["boyut"].ID, IntegerValue: int(SizID)})
}
if pr.Data.Color != "" {
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: AttributesMap["color"].ID, IntegerValue: int(pr.ColorOption.ID)})

301
repositories/scraper_lcw.go Normal file
View File

@ -0,0 +1,301 @@
package repositories
import (
"context"
"db_service/models"
helper "db_service/pkg"
"fmt"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/chromedp"
)
const (
LCW_URL = "https://www.lcwaikiki.com"
)
type LCWScraper struct {
link string
}
func NewLCWScraper(link string) LCWScraper {
return LCWScraper{link: link}
}
func (s LCWScraper) InitProductDetailParsing() (models.Product, error) {
product := models.Product{}
// load page with js
html, err := loadPage(s.link)
if err != nil {
return product, err
}
// convert to goquery document
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
fmt.Println(err)
return product, err
}
product = parseDocument(doc, true)
product.Vendor = "lcw"
return product, nil
}
// loadPage loads page using chromeDp
// ChromeDp waits until JS render complete. returns html string
func loadPage(url string) (string, error) {
var res string
ctx, cancel := chromedp.NewContext(context.Background())
defer cancel()
err := chromedp.Run(ctx,
chromedp.Navigate(url),
chromedp.ActionFunc(func(ctx context.Context) error {
node, err := dom.GetDocument().Do(ctx)
if err != nil {
return err
}
res, err = dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
return err
}),
)
if err != nil {
return res, err
}
return res, nil
}
//parseDocument parses document using goQuery
func parseDocument(doc *goquery.Document, primaryModel bool) models.Product {
product := &models.Product{
ColorVariants: &([]models.Product{}),
SizeVariants: &([]models.Variant{}),
}
doc.Find("meta").Each(func(i int, s *goquery.Selection) {
name, _ := s.Attr("name")
content, _ := s.Attr("content")
switch name {
case "ModelId":
product.ProductGroupID = content
case "OptionId":
product.Sku = "p-" + content
product.ProductNumber = "lcw-"+content
case "ProductCode":
product.ProductCode = content
case "ProductName":
product.Name = content
case "BrandName":
product.Brand = content
case "Gender":
product.Cinsiyet = content
case "description":
product.Description = content
case "Color":
product.Color = content
case "DiscountPrice_1":
product.Price.SellingPrice.Text = ""
product.Price.SellingPrice.Value = convertStrToFloat(content)
case "CashPrice_1":
product.Price.OriginalPrice.Value = convertStrToFloat(content)
product.Weight = "0"
// case "Size":
// fmt.Println("SIZE: ", content)
// case "SizeId":
// fmt.Println("SIZE_ID: ", content)
}
})
productLink, ok := doc.Find(".share-link").Attr("value")
if ok {
urlKey := strings.ReplaceAll(productLink, LCW_URL, " ")
product.URLKey = strings.TrimSpace(urlKey)
}
setupPrices(product, doc)
setupSizeVariants(product, doc)
setupSizeImages(product, doc)
// call color variants for only primary model
if primaryModel {
setupColorVariants(product, doc)
}
setupProductProperties(product, doc)
return *product
}
func setupPrices(product *models.Product, doc *goquery.Document) {
basketPrice := strings.Trim(doc.Find(".col-xs-12 .price-area > .campaign-discount-detail > .basket-discount").Text(), " \n")
basketPriceFloat := convertStrToFloat(basketPrice)
// check basket price discount
if basketPriceFloat != 0.0 {
product.Price.DiscountedPrice.Value = basketPriceFloat
}
}
func setupSizeVariants(product *models.Product, doc *goquery.Document) {
optionSizes := doc.Find(".option-size").Children()
for i := range optionSizes.Nodes {
optionSizes.Eq(i).Each(func(ii int, selection *goquery.Selection) {
sizeValue, _ := selection.Attr("size")
dataStock, _ := selection.Attr("data-stock")
key, _ := selection.Attr("key")
keyInt, err := strconv.Atoi(key)
if err != nil {
// ... handle error
// panic(err)
fmt.Println("error in generating keyInt for ItemNumber")
}
if len(dataStock) != 0 {
stock, _ := strconv.Atoi(dataStock)
if stock > 3 {
sizeVariant := models.Variant{
AttributeName: "Beden",
AttributeValue: sizeValue,
Stock: dataStock,
Sellable: true,
ItemNumber: keyInt,
Price: models.Price{
DiscountedPrice: models.PriceValue{
Text: "",
Value: product.Price.DiscountedPrice.Value,
},
SellingPrice: models.PriceValue{
Text: "",
Value: product.Price.SellingPrice.Value,
},
OriginalPrice: models.PriceValue{
Text: "",
Value: product.Price.OriginalPrice.Value,
},
},
}
isItemAdded := helper.IsLCWSizeVariantsAdded(*product.SizeVariants, sizeVariant)
if !isItemAdded {
*product.SizeVariants = append(*product.SizeVariants, sizeVariant)
}
}
}
})
}
}
//setupColorVariants gets color variant products.
func setupColorVariants(product *models.Product, doc *goquery.Document) {
colorVariants := make([]models.Product, 0)
doc.Find(".color-option").Each(func(i int, s *goquery.Selection) {
optionLink, ok := s.Attr("href")
if ok && !strings.Contains(optionLink, "javascript:") {
url := LCW_URL + optionLink
htmlPage, err := loadPage(url)
if err != nil {
return
}
newDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlPage))
if err != nil {
fmt.Println("err: ", err)
return
}
colorOption := parseDocument(newDoc, false)
colorVariants = append(colorVariants, colorOption)
}
})
product.ColorVariantCount = len(colorVariants)
*product.ColorVariants = append(*product.ColorVariants, colorVariants...)
}
func setupSizeImages(product *models.Product, doc *goquery.Document) {
// images := make([]string, 0)
doc.Find("img").Each(func(i int, s *goquery.Selection) {
if image, ok := s.Attr("smallimages"); ok {
isAdded := helper.IsImageAdded(product.Images,image)
if !isAdded {
product.Images = append(product.Images, image)
}
}
})
}
func setupProductProperties(product *models.Product, doc *goquery.Document) {
// get collapseOne
collapseOne := doc.Find("#collapseOne")
// get descriptions
collapseOne.Find("li").Each(func(i int, selection *goquery.Selection) {
desc := strings.TrimSpace(selection.Text())
if len(desc) != 0 {
desc := models.DescriptionModel{
Description: desc,
Bold: false,
}
// descriptions = append(descriptions, desc)
product.Descriptions = append(product.Descriptions, desc)
}
})
collapseOne.Find(".option-info").Each(func(i int, selection *goquery.Selection) {
selection.Find("p").Each(func(i int, s *goquery.Selection) {
text := strings.Trim(s.Text(), "")
if len(text) != 0 {
attr := strings.Split(text, ":")
attrKey, attrValue := strings.Trim(attr[0], " "), strings.Trim(attr[1], " ")
if len(attrKey) != 0 && len(attrValue) != 0 {
attribute := map[string]string{
strings.ToLower(attrKey): attrValue,
}
product.Attributes = append(product.Attributes, attribute)
}
}
})
})
}
func convertStrToFloat(price string) float64 {
// remove TL
priceStr := strings.ReplaceAll(price, " TL", "")
// remove .(dot)
priceStr = strings.ReplaceAll(priceStr, ".", "")
// replace ,(comma) with .(dot)
priceStr = strings.ReplaceAll(priceStr, ",", ".")
// remove whitespace
priceStr = strings.ReplaceAll(priceStr, " ", "")
priceFloat, err := strconv.ParseFloat(priceStr, 64)
if err != nil {
return 0
}
return priceFloat
}

91
vendor/modules.txt vendored
View File

@ -1,6 +1,81 @@
# github.com/PuerkitoBio/goquery v1.8.0
## explicit; go 1.13
github.com/PuerkitoBio/goquery
# github.com/andybalholm/cascadia v1.3.1
## explicit; go 1.16
github.com/andybalholm/cascadia
# github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24
## explicit; go 1.19
github.com/chromedp/cdproto
github.com/chromedp/cdproto/accessibility
github.com/chromedp/cdproto/animation
github.com/chromedp/cdproto/audits
github.com/chromedp/cdproto/backgroundservice
github.com/chromedp/cdproto/browser
github.com/chromedp/cdproto/cachestorage
github.com/chromedp/cdproto/cast
github.com/chromedp/cdproto/cdp
github.com/chromedp/cdproto/css
github.com/chromedp/cdproto/database
github.com/chromedp/cdproto/debugger
github.com/chromedp/cdproto/deviceorientation
github.com/chromedp/cdproto/dom
github.com/chromedp/cdproto/domdebugger
github.com/chromedp/cdproto/domsnapshot
github.com/chromedp/cdproto/domstorage
github.com/chromedp/cdproto/emulation
github.com/chromedp/cdproto/eventbreakpoints
github.com/chromedp/cdproto/fetch
github.com/chromedp/cdproto/headlessexperimental
github.com/chromedp/cdproto/heapprofiler
github.com/chromedp/cdproto/indexeddb
github.com/chromedp/cdproto/input
github.com/chromedp/cdproto/inspector
github.com/chromedp/cdproto/io
github.com/chromedp/cdproto/layertree
github.com/chromedp/cdproto/log
github.com/chromedp/cdproto/media
github.com/chromedp/cdproto/memory
github.com/chromedp/cdproto/network
github.com/chromedp/cdproto/overlay
github.com/chromedp/cdproto/page
github.com/chromedp/cdproto/performance
github.com/chromedp/cdproto/performancetimeline
github.com/chromedp/cdproto/profiler
github.com/chromedp/cdproto/runtime
github.com/chromedp/cdproto/security
github.com/chromedp/cdproto/serviceworker
github.com/chromedp/cdproto/storage
github.com/chromedp/cdproto/systeminfo
github.com/chromedp/cdproto/target
github.com/chromedp/cdproto/tethering
github.com/chromedp/cdproto/tracing
github.com/chromedp/cdproto/webaudio
github.com/chromedp/cdproto/webauthn
# github.com/chromedp/chromedp v0.8.7
## explicit; go 1.18
github.com/chromedp/chromedp
github.com/chromedp/chromedp/device
github.com/chromedp/chromedp/kb
# github.com/chromedp/sysutil v1.0.0
## explicit; go 1.15
github.com/chromedp/sysutil
# github.com/go-sql-driver/mysql v1.6.0
## explicit; go 1.10
github.com/go-sql-driver/mysql
# github.com/gobwas/httphead v0.1.0
## explicit; go 1.15
github.com/gobwas/httphead
# github.com/gobwas/pool v0.2.1
## explicit
github.com/gobwas/pool
github.com/gobwas/pool/internal/pmath
github.com/gobwas/pool/pbufio
github.com/gobwas/pool/pbytes
# github.com/gobwas/ws v1.1.0
## explicit; go 1.15
github.com/gobwas/ws
github.com/gobwas/ws/wsutil
# github.com/gorilla/mux v1.8.0
## explicit; go 1.12
github.com/gorilla/mux
@ -19,9 +94,25 @@ github.com/jinzhu/now
# github.com/joho/godotenv v1.4.0
## explicit; go 1.12
github.com/joho/godotenv
# github.com/josharian/intern v1.0.0
## explicit; go 1.5
github.com/josharian/intern
# github.com/leesper/couchdb-golang v1.2.1
## explicit
github.com/leesper/couchdb-golang
# github.com/mailru/easyjson v0.7.7
## explicit; go 1.12
github.com/mailru/easyjson
github.com/mailru/easyjson/buffer
github.com/mailru/easyjson/jlexer
github.com/mailru/easyjson/jwriter
# golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
## explicit; go 1.17
golang.org/x/net/html
golang.org/x/net/html/atom
# golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec
## explicit; go 1.17
golang.org/x/sys/unix
# gorm.io/driver/mysql v1.3.5
## explicit; go 1.14
gorm.io/driver/mysql