LCW IMPORTER GOSHULDY
This commit is contained in:
parent
313b33f238
commit
f5ad77ba52
|
|
@ -8,15 +8,75 @@ import (
|
|||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func ParseLink(w http.ResponseWriter, route *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
link := route.URL.Query().Get("url")
|
||||
|
||||
helper.Info("link: ", link)
|
||||
|
||||
if strings.Contains(link, "lcwaikiki.com"){
|
||||
ParseLinkLCW(link,w)
|
||||
}else {
|
||||
ParseLinkTY(link,w)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
func ParseLinkLCW(link string,w http.ResponseWriter) {
|
||||
start := time.Now()
|
||||
|
||||
linkParser := repositories.NewLCWScraper(link)
|
||||
|
||||
product, err := linkParser.InitProductDetailParsing()
|
||||
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
json.NewEncoder(w).Encode(map[string]string{
|
||||
"msg": err.Error(),
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
jsonProduct, err := linkParser.GetProductDetailWithOptions(product.ID, product.ProductGroupID)
|
||||
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return
|
||||
}
|
||||
/////////////////////////////
|
||||
importRepo, err := repositories.ImporterInstance()
|
||||
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
//wait until initialization data is loaded to memory
|
||||
importRepo.ImportWGroup.Wait()
|
||||
|
||||
if err = importRepo.UpdateOrCreate(jsonProduct).Error; err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]string{
|
||||
"msg": "Link parsed successfully",
|
||||
"productGroupId": strconv.Itoa(product.ID),
|
||||
})
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("end parse took %s", elapsed)
|
||||
return
|
||||
}
|
||||
|
||||
func ParseLinkTY(link string,w http.ResponseWriter) {
|
||||
start := time.Now()
|
||||
|
||||
linkParser := repositories.NewLinkParser(link)
|
||||
|
||||
product, err := linkParser.ParseLink()
|
||||
|
|
|
|||
12
go.mod
12
go.mod
|
|
@ -3,6 +3,9 @@ module db_service
|
|||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0
|
||||
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24
|
||||
github.com/chromedp/chromedp v0.8.7
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gosimple/slug v1.12.0
|
||||
github.com/joho/godotenv v1.4.0
|
||||
|
|
@ -12,8 +15,17 @@ require (
|
|||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/chromedp/sysutil v1.0.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.1.0 // indirect
|
||||
github.com/gosimple/unidecode v1.0.1 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
|
||||
)
|
||||
|
|
|
|||
32
go.sum
32
go.sum
|
|
@ -1,5 +1,21 @@
|
|||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24 h1:IKBU366KowxMcDpu9cgYEaXsiO0eBfPosTS0Krqfb8M=
|
||||
github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
||||
github.com/chromedp/chromedp v0.8.7 h1:dYOYc5ynTBzwSLOi+1IfgHwPr8r2BqV48l/RC+3OuJ0=
|
||||
github.com/chromedp/chromedp v0.8.7/go.mod h1:iL+ywnwk3eG3EVXV1ackXBMNzdEh3Ye/KHvQkq1KRKU=
|
||||
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
|
||||
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
|
||||
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
|
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gosimple/slug v1.12.0 h1:xzuhj7G7cGtd34NXnW/yF0l+AGNfWqwgh/IXgFy7dnc=
|
||||
|
|
@ -13,8 +29,24 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
|||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/leesper/couchdb-golang v1.2.1 h1:FqSaTxxT2mVRLbxGQVkZakRRoSzWhPmV8UEKYjA/GWc=
|
||||
github.com/leesper/couchdb-golang v1.2.1/go.mod h1:OU3FDAM3mazHx15oi8Hm+egTMneBUqepwnh0LuBSH54=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
|
||||
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
|
||||
gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
|
||||
gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
package models
|
||||
|
||||
|
||||
type LCWScrapDataModel struct {
|
||||
Id string
|
||||
Code string
|
||||
Name string
|
||||
OldPrice string
|
||||
Price string
|
||||
SizeVariants []SizeVariants
|
||||
ColorOptions []LCWScrapDataModel
|
||||
}
|
||||
|
||||
type DescriptionModel struct {
|
||||
Description string `json:"description"`
|
||||
Bold bool `json:"bold"`
|
||||
}
|
||||
|
|
@ -33,21 +33,17 @@ type Product struct {
|
|||
|
||||
type Price struct {
|
||||
ProfitMargin int `json:"profitMargin"`
|
||||
DiscountedPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"discountedPrice"`
|
||||
SellingPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"sellingPrice"`
|
||||
OriginalPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"originalPrice"`
|
||||
DiscountedPrice PriceValue `json:"discountedPrice"`
|
||||
SellingPrice PriceValue `json:"sellingPrice"`
|
||||
OriginalPrice PriceValue `json:"originalPrice"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
|
||||
type PriceValue struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
AttributeID int `json:"attributeId"`
|
||||
AttributeName string `json:"attributeName"`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
package models
|
||||
|
||||
type SizeVariants struct {
|
||||
Price float64 `json:"price"`
|
||||
Size string `json:"size"`
|
||||
Stock int `json:"stock"`
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package helper
|
|||
|
||||
import (
|
||||
"context"
|
||||
"db_service/models"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
|
|
@ -84,3 +85,11 @@ func GetCategoryWeight(slug string) string {
|
|||
|
||||
return doc["weight"].(string)
|
||||
}
|
||||
func IsLCWSizeVariantsAdded(items []models.Variant, keyItem models.Variant) bool {
|
||||
for _, option := range items {
|
||||
if option.ItemNumber == keyItem.ItemNumber {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package repositories
|
||||
|
||||
import helper "db_service/pkg"
|
||||
|
||||
type LinkParserLCW struct {
|
||||
link string
|
||||
}
|
||||
|
||||
func NewLinkParserLCW(link string) LinkParserLCW {
|
||||
return LinkParserLCW{link: link}
|
||||
}
|
||||
|
||||
func (l LinkParserLCW) ParseLink() (map[string]interface{}, error) {
|
||||
|
||||
helper.Info("link: ", l.link)
|
||||
|
||||
lcwScraper := NewLCWScraper(l.link)
|
||||
|
||||
product, err := lcwScraper.InitProductDetailParsing()
|
||||
if err != nil {
|
||||
helper.Error(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return product, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,296 @@
|
|||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"db_service/models"
|
||||
helper "db_service/pkg"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/chromedp/cdproto/dom"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
const (
|
||||
LCW_URL = "https://www.lcwaikiki.com"
|
||||
)
|
||||
|
||||
type LCWScraper struct {
|
||||
link string
|
||||
}
|
||||
|
||||
func NewLCWScraper(link string) LCWScraper {
|
||||
return LCWScraper{link: link}
|
||||
}
|
||||
|
||||
func (s LCWScraper) InitProductDetailParsing() (models.Product, error) {
|
||||
|
||||
product := models.Product{Vendor: "lcw"}
|
||||
|
||||
// load page with js
|
||||
html, err := loadPage(s.link)
|
||||
|
||||
if err != nil {
|
||||
return product, err
|
||||
}
|
||||
|
||||
// convert to goquery document
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return product, err
|
||||
}
|
||||
|
||||
product = parseDocument(doc, true)
|
||||
|
||||
return product, nil
|
||||
}
|
||||
|
||||
// loadPage loads page using chromeDp
|
||||
// ChromeDp waits until JS render complete. returns html string
|
||||
func loadPage(url string) (string, error) {
|
||||
var res string
|
||||
|
||||
ctx, cancel := chromedp.NewContext(context.Background())
|
||||
defer cancel()
|
||||
|
||||
err := chromedp.Run(ctx,
|
||||
chromedp.Navigate(url),
|
||||
chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
node, err := dom.GetDocument().Do(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res, err = dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
|
||||
return err
|
||||
}),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
//parseDocument parses document using goQuery
|
||||
func parseDocument(doc *goquery.Document, primaryModel bool) models.Product {
|
||||
|
||||
product := &models.Product{
|
||||
ColorVariants: &([]models.Product{}),
|
||||
SizeVariants: &([]models.Variant{}),
|
||||
}
|
||||
|
||||
doc.Find("meta").Each(func(i int, s *goquery.Selection) {
|
||||
name, _ := s.Attr("name")
|
||||
content, _ := s.Attr("content")
|
||||
switch name {
|
||||
case "ModelId":
|
||||
product.ProductGroupID = content
|
||||
case "OptionId":
|
||||
product.Sku = "p-" + content
|
||||
product.ProductNumber = content
|
||||
case "ProductCode":
|
||||
product.ProductCode = content
|
||||
case "ProductName":
|
||||
product.Name = content
|
||||
case "BrandName":
|
||||
product.Brand = content
|
||||
case "Gender":
|
||||
product.Cinsiyet = content
|
||||
case "description":
|
||||
product.Description = content
|
||||
case "Color":
|
||||
product.Color = content
|
||||
case "DiscountPrice_1":
|
||||
product.Price.SellingPrice.Text = ""
|
||||
product.Price.SellingPrice.Value = convertStrToFloat(content)
|
||||
case "CashPrice_1":
|
||||
product.Price.OriginalPrice.Value = convertStrToFloat(content)
|
||||
// case "Size":
|
||||
// fmt.Println("SIZE: ", content)
|
||||
// case "SizeId":
|
||||
// fmt.Println("SIZE_ID: ", content)
|
||||
}
|
||||
})
|
||||
|
||||
productLink, ok := doc.Find(".share-link").Attr("value")
|
||||
if ok {
|
||||
urlKey := strings.ReplaceAll(productLink, LCW_URL, " ")
|
||||
product.URLKey = strings.TrimSpace(urlKey)
|
||||
}
|
||||
|
||||
setupPrices(product, doc)
|
||||
setupSizeVariants(product, doc)
|
||||
setupSizeImages(product, doc)
|
||||
|
||||
// call color variants for only primary model
|
||||
if primaryModel {
|
||||
setupColorVariants(product, doc)
|
||||
}
|
||||
|
||||
setupProductProperties(product, doc)
|
||||
|
||||
return *product
|
||||
}
|
||||
|
||||
func setupPrices(product *models.Product, doc *goquery.Document) {
|
||||
|
||||
basketPrice := strings.Trim(doc.Find(".col-xs-12 .price-area > .campaign-discount-detail > .basket-discount").Text(), " \n")
|
||||
basketPriceFloat := convertStrToFloat(basketPrice)
|
||||
|
||||
// check basket price discount
|
||||
if basketPriceFloat != 0.0 {
|
||||
product.Price.DiscountedPrice.Value = basketPriceFloat
|
||||
}
|
||||
}
|
||||
|
||||
func setupSizeVariants(product *models.Product, doc *goquery.Document) {
|
||||
|
||||
optionSizes := doc.Find(".option-size").Children()
|
||||
for i := range optionSizes.Nodes {
|
||||
optionSizes.Eq(i).Each(func(ii int, selection *goquery.Selection) {
|
||||
sizeValue, _ := selection.Attr("size")
|
||||
dataStock, _ := selection.Attr("data-stock")
|
||||
key, _ := selection.Attr("key")
|
||||
|
||||
keyInt, err := strconv.Atoi(product.ProductNumber + key)
|
||||
if err != nil {
|
||||
// ... handle error
|
||||
// panic(err)
|
||||
fmt.Println("error in generating keyInt for ItemNumber")
|
||||
}
|
||||
|
||||
if len(dataStock) != 0 {
|
||||
stock, _ := strconv.Atoi(dataStock)
|
||||
|
||||
if stock > 3 {
|
||||
sizeVariant := models.Variant{
|
||||
AttributeName: "Beden",
|
||||
AttributeValue: sizeValue,
|
||||
Stock: dataStock,
|
||||
Sellable: true,
|
||||
ItemNumber: keyInt,
|
||||
Price: models.Price{
|
||||
DiscountedPrice: models.PriceValue{
|
||||
Text: "",
|
||||
Value: product.Price.DiscountedPrice.Value,
|
||||
},
|
||||
SellingPrice: models.PriceValue{
|
||||
Text: "",
|
||||
Value: product.Price.SellingPrice.Value,
|
||||
},
|
||||
OriginalPrice: models.PriceValue{
|
||||
Text: "",
|
||||
Value: product.Price.OriginalPrice.Value,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
isItemAdded := helper.IsLCWSizeVariantsAdded(*product.SizeVariants, sizeVariant)
|
||||
if !isItemAdded {
|
||||
*product.SizeVariants = append(*product.SizeVariants, sizeVariant)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
//setupColorVariants gets color variant products.
|
||||
func setupColorVariants(product *models.Product, doc *goquery.Document) {
|
||||
|
||||
colorVariants := make([]models.Product, 0)
|
||||
|
||||
doc.Find(".color-option").Each(func(i int, s *goquery.Selection) {
|
||||
optionLink, ok := s.Attr("href")
|
||||
if ok && !strings.Contains(optionLink, "javascript:") {
|
||||
url := LCW_URL + optionLink
|
||||
htmlPage, err := loadPage(url)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
newDoc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlPage))
|
||||
if err != nil {
|
||||
fmt.Println("err: ", err)
|
||||
return
|
||||
}
|
||||
|
||||
colorOption := parseDocument(newDoc, false)
|
||||
colorVariants = append(colorVariants, colorOption)
|
||||
}
|
||||
})
|
||||
|
||||
product.ColorVariantCount = len(colorVariants)
|
||||
*product.ColorVariants = append(*product.ColorVariants, colorVariants...)
|
||||
}
|
||||
|
||||
func setupSizeImages(product *models.Product, doc *goquery.Document) {
|
||||
|
||||
// images := make([]string, 0)
|
||||
|
||||
doc.Find("img").Each(func(i int, s *goquery.Selection) {
|
||||
if image, ok := s.Attr("smallimages"); ok {
|
||||
product.Images = append(product.Images, image)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func setupProductProperties(product *models.Product, doc *goquery.Document) {
|
||||
// get collapseOne
|
||||
collapseOne := doc.Find("#collapseOne")
|
||||
|
||||
// get descriptions
|
||||
collapseOne.Find("li").Each(func(i int, selection *goquery.Selection) {
|
||||
desc := strings.TrimSpace(selection.Text())
|
||||
if len(desc) != 0 {
|
||||
desc := models.DescriptionModel{
|
||||
Description: desc,
|
||||
Bold: false,
|
||||
}
|
||||
// descriptions = append(descriptions, desc)
|
||||
product.Descriptions = append(product.Descriptions, desc)
|
||||
}
|
||||
})
|
||||
|
||||
collapseOne.Find(".option-info").Each(func(i int, selection *goquery.Selection) {
|
||||
selection.Find("p").Each(func(i int, s *goquery.Selection) {
|
||||
text := strings.Trim(s.Text(), "")
|
||||
if len(text) != 0 {
|
||||
attr := strings.Split(text, ":")
|
||||
|
||||
attrKey, attrValue := strings.Trim(attr[0], " "), strings.Trim(attr[1], " ")
|
||||
|
||||
if len(attrKey) != 0 && len(attrValue) != 0 {
|
||||
attribute := map[string]string{
|
||||
strings.ToLower(attrKey): attrValue,
|
||||
}
|
||||
product.Attributes = append(product.Attributes, attribute)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func convertStrToFloat(price string) float64 {
|
||||
// remove TL
|
||||
priceStr := strings.ReplaceAll(price, " TL", "")
|
||||
// remove .(dot)
|
||||
priceStr = strings.ReplaceAll(priceStr, ".", "")
|
||||
// replace ,(comma) with .(dot)
|
||||
priceStr = strings.ReplaceAll(priceStr, ",", ".")
|
||||
// remove whitespace
|
||||
priceStr = strings.ReplaceAll(priceStr, " ", "")
|
||||
|
||||
priceFloat, err := strconv.ParseFloat(priceStr, 64)
|
||||
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return priceFloat
|
||||
}
|
||||
|
|
@ -1,6 +1,81 @@
|
|||
# github.com/PuerkitoBio/goquery v1.8.0
|
||||
## explicit; go 1.13
|
||||
github.com/PuerkitoBio/goquery
|
||||
# github.com/andybalholm/cascadia v1.3.1
|
||||
## explicit; go 1.16
|
||||
github.com/andybalholm/cascadia
|
||||
# github.com/chromedp/cdproto v0.0.0-20230120182703-ecee3ffd2a24
|
||||
## explicit; go 1.19
|
||||
github.com/chromedp/cdproto
|
||||
github.com/chromedp/cdproto/accessibility
|
||||
github.com/chromedp/cdproto/animation
|
||||
github.com/chromedp/cdproto/audits
|
||||
github.com/chromedp/cdproto/backgroundservice
|
||||
github.com/chromedp/cdproto/browser
|
||||
github.com/chromedp/cdproto/cachestorage
|
||||
github.com/chromedp/cdproto/cast
|
||||
github.com/chromedp/cdproto/cdp
|
||||
github.com/chromedp/cdproto/css
|
||||
github.com/chromedp/cdproto/database
|
||||
github.com/chromedp/cdproto/debugger
|
||||
github.com/chromedp/cdproto/deviceorientation
|
||||
github.com/chromedp/cdproto/dom
|
||||
github.com/chromedp/cdproto/domdebugger
|
||||
github.com/chromedp/cdproto/domsnapshot
|
||||
github.com/chromedp/cdproto/domstorage
|
||||
github.com/chromedp/cdproto/emulation
|
||||
github.com/chromedp/cdproto/eventbreakpoints
|
||||
github.com/chromedp/cdproto/fetch
|
||||
github.com/chromedp/cdproto/headlessexperimental
|
||||
github.com/chromedp/cdproto/heapprofiler
|
||||
github.com/chromedp/cdproto/indexeddb
|
||||
github.com/chromedp/cdproto/input
|
||||
github.com/chromedp/cdproto/inspector
|
||||
github.com/chromedp/cdproto/io
|
||||
github.com/chromedp/cdproto/layertree
|
||||
github.com/chromedp/cdproto/log
|
||||
github.com/chromedp/cdproto/media
|
||||
github.com/chromedp/cdproto/memory
|
||||
github.com/chromedp/cdproto/network
|
||||
github.com/chromedp/cdproto/overlay
|
||||
github.com/chromedp/cdproto/page
|
||||
github.com/chromedp/cdproto/performance
|
||||
github.com/chromedp/cdproto/performancetimeline
|
||||
github.com/chromedp/cdproto/profiler
|
||||
github.com/chromedp/cdproto/runtime
|
||||
github.com/chromedp/cdproto/security
|
||||
github.com/chromedp/cdproto/serviceworker
|
||||
github.com/chromedp/cdproto/storage
|
||||
github.com/chromedp/cdproto/systeminfo
|
||||
github.com/chromedp/cdproto/target
|
||||
github.com/chromedp/cdproto/tethering
|
||||
github.com/chromedp/cdproto/tracing
|
||||
github.com/chromedp/cdproto/webaudio
|
||||
github.com/chromedp/cdproto/webauthn
|
||||
# github.com/chromedp/chromedp v0.8.7
|
||||
## explicit; go 1.18
|
||||
github.com/chromedp/chromedp
|
||||
github.com/chromedp/chromedp/device
|
||||
github.com/chromedp/chromedp/kb
|
||||
# github.com/chromedp/sysutil v1.0.0
|
||||
## explicit; go 1.15
|
||||
github.com/chromedp/sysutil
|
||||
# github.com/go-sql-driver/mysql v1.6.0
|
||||
## explicit; go 1.10
|
||||
github.com/go-sql-driver/mysql
|
||||
# github.com/gobwas/httphead v0.1.0
|
||||
## explicit; go 1.15
|
||||
github.com/gobwas/httphead
|
||||
# github.com/gobwas/pool v0.2.1
|
||||
## explicit
|
||||
github.com/gobwas/pool
|
||||
github.com/gobwas/pool/internal/pmath
|
||||
github.com/gobwas/pool/pbufio
|
||||
github.com/gobwas/pool/pbytes
|
||||
# github.com/gobwas/ws v1.1.0
|
||||
## explicit; go 1.15
|
||||
github.com/gobwas/ws
|
||||
github.com/gobwas/ws/wsutil
|
||||
# github.com/gorilla/mux v1.8.0
|
||||
## explicit; go 1.12
|
||||
github.com/gorilla/mux
|
||||
|
|
@ -19,9 +94,25 @@ github.com/jinzhu/now
|
|||
# github.com/joho/godotenv v1.4.0
|
||||
## explicit; go 1.12
|
||||
github.com/joho/godotenv
|
||||
# github.com/josharian/intern v1.0.0
|
||||
## explicit; go 1.5
|
||||
github.com/josharian/intern
|
||||
# github.com/leesper/couchdb-golang v1.2.1
|
||||
## explicit
|
||||
github.com/leesper/couchdb-golang
|
||||
# github.com/mailru/easyjson v0.7.7
|
||||
## explicit; go 1.12
|
||||
github.com/mailru/easyjson
|
||||
github.com/mailru/easyjson/buffer
|
||||
github.com/mailru/easyjson/jlexer
|
||||
github.com/mailru/easyjson/jwriter
|
||||
# golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
|
||||
## explicit; go 1.17
|
||||
golang.org/x/net/html
|
||||
golang.org/x/net/html/atom
|
||||
# golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec
|
||||
## explicit; go 1.17
|
||||
golang.org/x/sys/unix
|
||||
# gorm.io/driver/mysql v1.3.5
|
||||
## explicit; go 1.14
|
||||
gorm.io/driver/mysql
|
||||
|
|
|
|||
Loading…
Reference in New Issue