first commit
This commit is contained in:
parent
aea653ed18
commit
ded68e5b60
|
|
@ -0,0 +1,16 @@
|
|||
couch_db_source=http://admin:adminadmin@192.168.1.2:5984/
|
||||
|
||||
db_trendyol=trendyol_db_v2
|
||||
db_ty_categories=ty_categories
|
||||
|
||||
remote_url=https://sarga.com.tm/api/
|
||||
descendant_category_url=descendant-categories
|
||||
|
||||
db=test_baza
|
||||
du=merdan
|
||||
dp=qazwsx12
|
||||
da=localhost
|
||||
dp=3306
|
||||
#database_url ="merdan:qazwsx12@tcp(127.0.0.1:3306)/test_baza?parseTime=true"
|
||||
database_url ="orient:orient@tcp(192.168.1.2:3306)/bagisto_sarga?parseTime=true"
|
||||
couchdb_url = "http://admin:adminadmin@192.168.1.2:5984/"
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
.env
|
||||
db_service.exe
|
||||
/vendor
|
||||
.vscode
|
||||
db_service
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
gm "db_service/gorm_models"
|
||||
"db_service/models"
|
||||
helper "db_service/pkg"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func dbService() error {
|
||||
fmt.Println("dbService started")
|
||||
|
||||
tayyarla()
|
||||
|
||||
start := time.Now()
|
||||
|
||||
var resultingChannel = make(chan gm.Category)
|
||||
var producer_wg sync.WaitGroup
|
||||
var consumer_wg sync.WaitGroup
|
||||
|
||||
// consumer
|
||||
for c := 0; c < len(mainCategories); c++ {
|
||||
producer_wg.Add(1)
|
||||
go func() {
|
||||
defer consumer_wg.Done()
|
||||
|
||||
// WaitGroup for iterate functions
|
||||
const max = 20
|
||||
var wg sync.WaitGroup
|
||||
|
||||
responses := make(chan bool, max)
|
||||
|
||||
for item := range resultingChannel {
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
db := "ty_db_" + item.Translations[0].Slug
|
||||
|
||||
dbExists := helper.CheckDBExists("http://admin:adminadmin@localhost:5984/" + db)
|
||||
|
||||
if dbExists {
|
||||
go iterate(db, responses, func() { wg.Done() })
|
||||
}
|
||||
}
|
||||
|
||||
// close iterate function responses
|
||||
go func() {
|
||||
defer close(responses)
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
for response := range responses {
|
||||
fmt.Println("response", response)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// producer
|
||||
for _, cat := range mainCategories {
|
||||
consumer_wg.Add(1)
|
||||
go func(category gm.Category) {
|
||||
defer producer_wg.Done()
|
||||
resultingChannel <- category
|
||||
}(cat)
|
||||
}
|
||||
|
||||
producer_wg.Wait()
|
||||
|
||||
close(resultingChannel)
|
||||
|
||||
consumer_wg.Wait()
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Println("DB Time Elapsed:", elapsed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// func init() {
|
||||
// panic("unimplemented")
|
||||
// }
|
||||
|
||||
// iterate categories
|
||||
func iterate(db string, finished chan<- bool, onExit func()) {
|
||||
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
defer onExit()
|
||||
|
||||
// start := time.Now()
|
||||
|
||||
// get total product count from couch database
|
||||
totalDocCount := getTotalDocumentCount(db)
|
||||
|
||||
if totalDocCount == 0 {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
// total document counter
|
||||
i := 0
|
||||
|
||||
for i < totalDocCount {
|
||||
|
||||
cdbTime := time.Now()
|
||||
product, err := getDataFromCouchDb(i, db)
|
||||
i++
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
//TODO: insert product to mysql
|
||||
fmt.Println("product: ", product.Name, " db: ", db)
|
||||
|
||||
log.Println("Couch DB. Time elapsed:", time.Since(cdbTime), " DB: ", db)
|
||||
|
||||
// gorm.AutoMigrate(&product)
|
||||
|
||||
// // if there is an error inserting, handle it
|
||||
// if err != nil {
|
||||
// gorm.Close()
|
||||
// panic(err.Error())
|
||||
// }
|
||||
// be careful deferring Queries if you are using transactions
|
||||
// gorm.Close()
|
||||
|
||||
}
|
||||
|
||||
finished <- true
|
||||
}
|
||||
|
||||
// getTotalDocumentCount gets the total number of documents and returns
|
||||
func getTotalDocumentCount(db string) int {
|
||||
|
||||
var response models.DBDocCountResponse
|
||||
|
||||
url := os.Getenv("couch_db_source") + db
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "", true)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return 0
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return 0
|
||||
}
|
||||
|
||||
return response.DocCount
|
||||
}
|
||||
|
||||
// getDataFromCouchDb gets data from local couch db
|
||||
func getDataFromCouchDb(i int, db string) (models.Product, error) {
|
||||
var model models.Product
|
||||
var response models.BagistoModelResponse
|
||||
|
||||
url := os.Getenv("couch_db_source") + db + fmt.Sprintf("/_all_docs?include_docs=true&limit=1&skip=%v", i)
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "", true)
|
||||
if err != nil {
|
||||
return model, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return model, err
|
||||
}
|
||||
|
||||
return response.Rows[0].Doc, nil
|
||||
}
|
||||
|
||||
//https://gorm.io/docs/query.html
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var garash, garash2 sync.WaitGroup
|
||||
|
||||
func dene() {
|
||||
|
||||
garash.Add(1)
|
||||
|
||||
go func() {
|
||||
defer garash.Done()
|
||||
time.Sleep(5 * time.Second)
|
||||
fmt.Println("5 sekunt garashdyrmakchy")
|
||||
}()
|
||||
garash2.Add(1)
|
||||
go ishlet()
|
||||
garash2.Wait()
|
||||
fmt.Println("gutardym")
|
||||
}
|
||||
|
||||
func ishlet() {
|
||||
defer garash2.Done()
|
||||
garash.Wait()
|
||||
fmt.Println("garashyp boldym")
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
module db_service
|
||||
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/gosimple/slug v1.12.0
|
||||
github.com/jinzhu/gorm v1.9.16
|
||||
github.com/joho/godotenv v1.4.0
|
||||
gorm.io/driver/mysql v1.3.5
|
||||
gorm.io/gorm v1.23.8
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/gosimple/unidecode v1.0.1 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/gosimple/slug v1.12.0 h1:xzuhj7G7cGtd34NXnW/yF0l+AGNfWqwgh/IXgFy7dnc=
|
||||
github.com/gosimple/slug v1.12.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
|
||||
github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
|
||||
github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
|
||||
github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o=
|
||||
github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
|
||||
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
|
||||
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY=
|
||||
gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
|
||||
gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
|
||||
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package gorm_models
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/gosimple/slug"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type Brand struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
Name string
|
||||
Code string
|
||||
Status bool
|
||||
Categories []Category `gorm:"many2many:category_brands;"`
|
||||
}
|
||||
|
||||
func FindOrCreateBrand(db *gorm.DB, brand string, categories []Category) Brand {
|
||||
|
||||
var brandObject Brand
|
||||
|
||||
if brand != "" {
|
||||
|
||||
code := slug.Make(brand)
|
||||
|
||||
err := db.FirstOrCreate(&brandObject, Brand{Name: brand, Code: code}).Error
|
||||
// err := db.Model(&Brand{}).Find(&brandObject).Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
} else {
|
||||
db.Model(&brandObject).Association("Categories").Append(categories)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// var brandObject Brand
|
||||
|
||||
return brandObject
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package gorm_models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type CategoryTranslation struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Slug string
|
||||
CategoryID uint
|
||||
MetaDescription string
|
||||
MetaKeywords string
|
||||
Locale string
|
||||
}
|
||||
|
||||
type Category struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Status int8
|
||||
Position int
|
||||
ParentId uint
|
||||
DisplayMode string
|
||||
Translations []CategoryTranslation
|
||||
}
|
||||
|
||||
func GetMainCategories(db *gorm.DB) []Category {
|
||||
var categories []Category
|
||||
err := db.Model(&Category{}).Preload("Translations").Find(&categories, "status=1 AND parent_id=1 AND display_mode!='promotion'").Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
||||
}
|
||||
|
||||
return categories
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package gorm_models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type AttributeOption struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
AttributeID uint
|
||||
AdminName string
|
||||
}
|
||||
|
||||
type Attribute struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Code string
|
||||
AttributeOptions []AttributeOption
|
||||
}
|
||||
|
||||
type AttributeFamily struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Code string
|
||||
Groups []AttributeGroup
|
||||
}
|
||||
|
||||
type AttributeGroup struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Name string
|
||||
AttributeFamilyID uint
|
||||
Attributes []Attribute `gorm:"many2many:attribute_group_mappings;"`
|
||||
}
|
||||
|
||||
func GetFamilies(db *gorm.DB) []AttributeFamily {
|
||||
var families []AttributeFamily
|
||||
err := db.Model(&AttributeFamily{}).Preload("Groups").Order("id ASC").Find(&families).Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
||||
} else {
|
||||
log.Println("famililar alyndy", len(families))
|
||||
}
|
||||
|
||||
return families
|
||||
}
|
||||
|
||||
func GetGroups(db *gorm.DB) []AttributeGroup {
|
||||
var groups []AttributeGroup
|
||||
err := db.Model(&AttributeGroup{}).Preload("Family").Find(&groups).Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func GetAttributes(db *gorm.DB) []Attribute {
|
||||
var attributes []Attribute
|
||||
err := db.Model(&Attribute{}).Find(&attributes).Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
||||
}
|
||||
return attributes
|
||||
}
|
||||
|
||||
func GetAttributeOption(db *gorm.DB, attrbuteID uint, value string) AttributeOption {
|
||||
var option AttributeOption
|
||||
err := db.FirstOrCreate(&option, AttributeOption{AttributeID: attrbuteID, AdminName: value}).Error
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
||||
}
|
||||
return option
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package gorm_models
|
||||
|
||||
import "time"
|
||||
|
||||
type Product struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
Sku string
|
||||
Type string
|
||||
ParentID uint `sql:"DEFAULT:NULL"`
|
||||
AttributeFamilyID uint
|
||||
AttributeFamily AttributeFamily
|
||||
BrandID uint `sql:"DEFAULT:NULL"`
|
||||
Brand Brand
|
||||
Images []ProductImage
|
||||
Categories []Category `gorm:"many2many:product_categories;"`
|
||||
AttributeValues []ProductAttributeValue
|
||||
}
|
||||
|
||||
// $main_attributes = [
|
||||
//
|
||||
// 'sku' => $parentProduct->sku,
|
||||
// 'product_number' => $data['product_number'],
|
||||
// 'name' => $data['name'],
|
||||
// 'weight' => $data['weight'] ?? 0.45,
|
||||
// 'source' => $data['url_key'],
|
||||
// 'status' => 1,
|
||||
// 'visible_individually' => 1,
|
||||
// 'url_key' => $parentProduct->sku,
|
||||
// 'short_description' => $desc,
|
||||
// 'description' => $desc,
|
||||
// 'favoritesCount' => $data['favorite_count']
|
||||
//
|
||||
// ];
|
||||
type ProductFlat struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
Sku string
|
||||
ProductNumber string
|
||||
Name string
|
||||
Weight float64 `gorm:"type:decimal(12,4)"`
|
||||
Status bool
|
||||
Source string
|
||||
VisibleIndividually bool
|
||||
UrlKey string
|
||||
ShortDescription string
|
||||
Description string
|
||||
FavoritesCount uint
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
Product Product
|
||||
}
|
||||
|
||||
type ProductImage struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
Type string
|
||||
Path string
|
||||
ProductID uint
|
||||
}
|
||||
|
||||
type ProductAttributeValue struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
Locale string
|
||||
Channel string
|
||||
ProductID uint
|
||||
AttributeID uint
|
||||
TextValue string
|
||||
BooleanValue bool
|
||||
IntegerValue int
|
||||
FloatValue float64 `gorm:"type:decimal(12,4)"`
|
||||
}
|
||||
|
||||
type ProductSuperAttribute struct {
|
||||
ProductID uint
|
||||
AttributeID uint
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package gorm_models
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type MarketplaceSeller struct {
|
||||
ID uint `gorm:"primaryKey"`
|
||||
Url string
|
||||
}
|
||||
|
||||
func GetSellers(db *gorm.DB) []MarketplaceSeller {
|
||||
var sellers []MarketplaceSeller
|
||||
err := db.Model(&MarketplaceSeller{}).Find(&sellers).Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
|
||||
}
|
||||
return sellers
|
||||
}
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
gm "db_service/gorm_models"
|
||||
cb "db_service/models"
|
||||
helper "db_service/pkg"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var mainCategories []gm.Category
|
||||
var baza *gorm.DB
|
||||
var resultingChannel = make(chan gm.Category)
|
||||
var producer_wg sync.WaitGroup
|
||||
var consumer_wg sync.WaitGroup
|
||||
|
||||
func insertToBaza(product cb.Product) bool {
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func tayyarla() {
|
||||
|
||||
// baza, err := gorm.Open("mysql", os.Getenv("database_url"))
|
||||
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
|
||||
// mainCategories = gm.GetMainCategories(baza)
|
||||
|
||||
}
|
||||
|
||||
func bashlat() error {
|
||||
|
||||
fmt.Println("Insert service starts")
|
||||
start := time.Now()
|
||||
tayyarla()
|
||||
|
||||
fmt.Println("consuming")
|
||||
for c := 0; c < len(mainCategories); c++ {
|
||||
producer_wg.Add(1)
|
||||
go func() {
|
||||
defer consumer_wg.Done()
|
||||
|
||||
// WaitGroup for iterate functions
|
||||
const max = 20
|
||||
var wg sync.WaitGroup
|
||||
|
||||
responses := make(chan bool, max)
|
||||
|
||||
for item := range resultingChannel {
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
db := "ty_db_" + item.Translations[0].Slug
|
||||
|
||||
dbExists := helper.CheckDBExists(os.Getenv("couchdb_url") + db)
|
||||
|
||||
if dbExists {
|
||||
go iterateCategories(db, responses, func() { wg.Done() })
|
||||
}
|
||||
}
|
||||
|
||||
// close iterate function responses
|
||||
go func() {
|
||||
defer close(responses)
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
for response := range responses {
|
||||
fmt.Println("response", response)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
fmt.Println("producing")
|
||||
for _, cat := range mainCategories {
|
||||
consumer_wg.Add(1)
|
||||
go func(category gm.Category) {
|
||||
defer producer_wg.Done()
|
||||
resultingChannel <- category
|
||||
}(cat)
|
||||
}
|
||||
producer_wg.Wait()
|
||||
|
||||
close(resultingChannel)
|
||||
|
||||
consumer_wg.Wait()
|
||||
|
||||
elapsed := time.Since(start)
|
||||
|
||||
log.Println("DB Time Elapsed:", elapsed)
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// func consumer() {
|
||||
// fmt.Println("consuming")
|
||||
// for c := 0; c < len(mainCategories); c++ {
|
||||
// producer_wg.Add(1)
|
||||
// go func() {
|
||||
// defer consumer_wg.Done()
|
||||
|
||||
// // WaitGroup for iterate functions
|
||||
// const max = 20
|
||||
// var wg sync.WaitGroup
|
||||
|
||||
// responses := make(chan bool, max)
|
||||
|
||||
// for item := range resultingChannel {
|
||||
|
||||
// wg.Add(1)
|
||||
|
||||
// db := "ty_db_" + item.Translations[0].Slug
|
||||
|
||||
// dbExists := helper.CheckDBExists(os.Getenv("couchdb_url") + db)
|
||||
|
||||
// if dbExists {
|
||||
// go iterateCategories(db, responses, func() { wg.Done() })
|
||||
// }
|
||||
// }
|
||||
|
||||
// // close iterate function responses
|
||||
// go func() {
|
||||
// defer close(responses)
|
||||
// wg.Wait()
|
||||
// }()
|
||||
|
||||
// for response := range responses {
|
||||
// fmt.Println("response", response)
|
||||
// }
|
||||
// }()
|
||||
// }
|
||||
// }
|
||||
|
||||
// func producer() {
|
||||
// fmt.Println("producing")
|
||||
// for _, cat := range mainCategories {
|
||||
// consumer_wg.Add(1)
|
||||
// go func(category gm.Category) {
|
||||
// defer producer_wg.Done()
|
||||
// resultingChannel <- category
|
||||
// }(cat)
|
||||
// }
|
||||
// producer_wg.Wait()
|
||||
|
||||
// close(resultingChannel)
|
||||
|
||||
// consumer_wg.Wait()
|
||||
// }
|
||||
|
||||
// iterate categories
|
||||
func iterateCategories(db string, finished chan<- bool, onExit func()) {
|
||||
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
defer onExit()
|
||||
|
||||
// start := time.Now()
|
||||
|
||||
// get total product count from couch database
|
||||
totalDocCount := getTotalDocumentCount(db)
|
||||
|
||||
if totalDocCount == 0 {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
// total document counter
|
||||
i := 0
|
||||
|
||||
for i < totalDocCount {
|
||||
|
||||
cdbTime := time.Now()
|
||||
product, err := getDataFromCouchDb(i, db)
|
||||
i++
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
|
||||
//TODO: insert product to mysql
|
||||
fmt.Println("product: ", product.Name, " db: ", db)
|
||||
|
||||
log.Println("Couch DB. Time elapsed:", time.Since(cdbTime), " DB: ", db)
|
||||
|
||||
// gorm.AutoMigrate(&product)
|
||||
|
||||
// // if there is an error inserting, handle it
|
||||
// if err != nil {
|
||||
// gorm.Close()
|
||||
// panic(err.Error())
|
||||
// }
|
||||
// be careful deferring Queries if you are using transactions
|
||||
// gorm.Close()
|
||||
|
||||
}
|
||||
|
||||
finished <- true
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"db_service/test"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
||||
log.Println("init function")
|
||||
|
||||
// init .env
|
||||
err := godotenv.Load()
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
// err := bashlat()
|
||||
start := time.Now()
|
||||
|
||||
r := new(big.Int)
|
||||
fmt.Println("start import", r.Binomial(1000, 10))
|
||||
|
||||
err := test.StartTest()
|
||||
if err != nil {
|
||||
fmt.Println("err: ", err.Error())
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("end import took %s", elapsed)
|
||||
// dene()
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package models
|
||||
|
||||
type Category struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
TrendyolURL string `json:"trendyol_url"`
|
||||
LcwURL string `json:"lcw_url"`
|
||||
DisplayMode string `json:"display_mode"`
|
||||
ImageURL interface{} `json:"image_url"`
|
||||
CategoryIconPath interface{} `json:"category_icon_path"`
|
||||
ProductLimit int `json:"product_limit"`
|
||||
Children []Category
|
||||
}
|
||||
|
||||
type BagistoResponse struct {
|
||||
Data []Category
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
package models
|
||||
|
||||
type DBDocCountResponse struct {
|
||||
DocCount int `json:"doc_count"`
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
package models
|
||||
|
||||
type Product struct {
|
||||
Attributes []map[string]string `json:"attributes"`
|
||||
Brand string `json:"brand"`
|
||||
Categories []int `json:"categories"`
|
||||
Cinsiyet string `json:"cinsiyet"`
|
||||
Color string `json:"color"`
|
||||
ColorVariantCount int `json:"color_variant_count"`
|
||||
ColorVariants []Product `json:"color_variants"`
|
||||
Description string `json:"description"`
|
||||
IsSellable bool `json:"sellable"`
|
||||
FavoriteCount int `json:"favorite_count"`
|
||||
Descriptions []struct {
|
||||
Description string `json:"description"`
|
||||
Bold bool `json:"bold"`
|
||||
} `json:"descriptions"`
|
||||
Images []string `json:"images"`
|
||||
Name string `json:"name"`
|
||||
NameWithProductCode string `json:"name_with_product_code"`
|
||||
Price Price `json:"price"`
|
||||
ProductCode string `json:"product_code"`
|
||||
ProductGroupID string `json:"product_group_id"`
|
||||
ProductNumber string `json:"product_number"`
|
||||
ShortDescription string `json:"short_description"`
|
||||
SizeVariants []Variant `json:"size_variants"`
|
||||
Sku string `json:"sku"`
|
||||
Stock interface{} `json:"stock"`
|
||||
URLKey string `json:"url_key"`
|
||||
Vendor string `json:"vendor"`
|
||||
Weight string `json:"weight"`
|
||||
}
|
||||
|
||||
type Price struct {
|
||||
ProfitMargin int `json:"profitMargin"`
|
||||
DiscountedPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"discountedPrice"`
|
||||
SellingPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"sellingPrice"`
|
||||
OriginalPrice struct {
|
||||
Text string `json:"text"`
|
||||
Value float64 `json:"value"`
|
||||
} `json:"originalPrice"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
AttributeID int `json:"attributeId"`
|
||||
AttributeName string `json:"attributeName"`
|
||||
AttributeType string `json:"attributeType"`
|
||||
AttributeValue string `json:"attributeValue"`
|
||||
Stamps []interface{} `json:"stamps"`
|
||||
Price Price `json:"price"`
|
||||
FulfilmentType string `json:"fulfilmentType"`
|
||||
AttributeBeautifiedValue string `json:"attributeBeautifiedValue"`
|
||||
IsWinner bool `json:"isWinner"`
|
||||
ListingID string `json:"listingId"`
|
||||
Stock interface{} `json:"stock"`
|
||||
Sellable bool `json:"sellable"`
|
||||
AvailableForClaim bool `json:"availableForClaim"`
|
||||
Barcode string `json:"barcode"`
|
||||
ItemNumber int `json:"itemNumber"`
|
||||
DiscountedPriceInfo string `json:"discountedPriceInfo"`
|
||||
HasCollectable bool `json:"hasCollectable"`
|
||||
RushDeliveryMerchantListingExist bool `json:"rushDeliveryMerchantListingExist"`
|
||||
LowerPriceMerchantListingExist bool `json:"lowerPriceMerchantListingExist"`
|
||||
}
|
||||
type Row struct {
|
||||
Key string `json:"key"`
|
||||
Value map[string]interface{} `json:"value"`
|
||||
Doc Product
|
||||
}
|
||||
type BagistoModelResponse struct {
|
||||
Rows []Row `json:"rows"`
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package helper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func CheckDBExists(endpoint string) bool {
|
||||
|
||||
const ConnectMaxWaitTime = 30 * time.Second
|
||||
const RequestMaxWaitTime = 60 * time.Second
|
||||
|
||||
client := http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: ConnectMaxWaitTime,
|
||||
}).DialContext,
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), RequestMaxWaitTime)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
|
||||
req.Proto = "HTTP/1.0"
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return false
|
||||
}
|
||||
|
||||
q := req.URL.Query()
|
||||
req.URL.RawQuery = q.Encode()
|
||||
req.Header.Set("Connection", "close")
|
||||
req.Close = true
|
||||
client.CloseIdleConnections()
|
||||
|
||||
response, err := client.Do(req)
|
||||
|
||||
// handle timeout
|
||||
if e, ok := err.(net.Error); ok && e.Timeout() {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
log.Println(err.Error())
|
||||
return false
|
||||
} else if err != nil {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
log.Println(err.Error())
|
||||
return false
|
||||
}
|
||||
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
|
||||
if response.StatusCode == http.StatusOK {
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
package helper
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func SendRequest(method string, endpoint string, values []byte, authKey string, isCouchDbReq bool) ([]byte, error) {
|
||||
|
||||
const ConnectMaxWaitTime = 30 * time.Second
|
||||
const RequestMaxWaitTime = 60 * time.Second
|
||||
|
||||
client := http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: ConnectMaxWaitTime,
|
||||
}).DialContext,
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), RequestMaxWaitTime)
|
||||
defer cancel()
|
||||
|
||||
emptyBody := []byte{}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(values))
|
||||
req.Proto = "HTTP/2.0"
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return emptyBody, err
|
||||
}
|
||||
|
||||
q := req.URL.Query()
|
||||
req.URL.RawQuery = q.Encode()
|
||||
req.Header.Set("Connection", "close")
|
||||
req.Header.Add("Authorization", authKey)
|
||||
req.Close = true
|
||||
client.CloseIdleConnections()
|
||||
|
||||
response, err := client.Do(req)
|
||||
|
||||
if e, ok := err.(net.Error); ok && e.Timeout() {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
log.Println(err.Error())
|
||||
return emptyBody, err
|
||||
} else if err != nil {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
log.Println(err.Error())
|
||||
return emptyBody, err
|
||||
}
|
||||
|
||||
// 200 OK
|
||||
// 201 Created
|
||||
// 202 Accepted
|
||||
// 404 Not Found
|
||||
// 409 Conflict
|
||||
|
||||
if isCouchDbReq {
|
||||
// fmt.Printf("responseStatusCode: %d\n", response.StatusCode)
|
||||
if response.StatusCode == http.StatusNotFound {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
return emptyBody, nil
|
||||
}
|
||||
// TODO: handle conflict error
|
||||
} else {
|
||||
if response.StatusCode != http.StatusOK {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
err := fmt.Errorf("response: code: %d, body: %v", response.StatusCode, response.Body)
|
||||
log.Println(err.Error())
|
||||
return emptyBody, err
|
||||
}
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
return emptyBody, nil
|
||||
}
|
||||
|
||||
// if you call it in loop, don't use defer
|
||||
response.Body.Close()
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,509 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
gm "db_service/gorm_models"
|
||||
"db_service/models"
|
||||
helper "db_service/pkg"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var mainCategories []gm.Category
|
||||
var baza *gorm.DB
|
||||
var mainImportWG, famAndSellerWG sync.WaitGroup
|
||||
var families []gm.AttributeFamily
|
||||
var sellers []gm.MarketplaceSeller
|
||||
var attributesMap = make(map[string]gm.Attribute)
|
||||
|
||||
func StartTest() error {
|
||||
baza, err := gorm.Open(mysql.Open(os.Getenv("database_url")), &gorm.Config{})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//get categories from mysql on main thread
|
||||
mainCategories = gm.GetMainCategories(baza)
|
||||
|
||||
famAndSellerWG.Add(3)
|
||||
//get attribute families from mysql with gorutine
|
||||
go func() {
|
||||
defer famAndSellerWG.Done()
|
||||
families = gm.GetFamilies(baza)
|
||||
}()
|
||||
|
||||
//get sellers families from mysql with gorutine
|
||||
go func() {
|
||||
defer famAndSellerWG.Done()
|
||||
sellers = gm.GetSellers(baza)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer famAndSellerWG.Done()
|
||||
var attributes = gm.GetAttributes(baza)
|
||||
for _, atrattribute := range attributes {
|
||||
attributesMap[atrattribute.Code] = atrattribute
|
||||
}
|
||||
}()
|
||||
|
||||
mainImportWG.Add(len(mainCategories))
|
||||
|
||||
for _, element := range mainCategories {
|
||||
slug := element.Translations[0].Slug
|
||||
|
||||
go startImport("ty_db_"+slug, baza)
|
||||
|
||||
// fmt.Println(<-result)
|
||||
}
|
||||
|
||||
mainImportWG.Wait()
|
||||
fmt.Println("Terminating Program")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func startImport(dbName string, db *gorm.DB) {
|
||||
defer mainImportWG.Done()
|
||||
|
||||
dbExists := helper.CheckDBExists(os.Getenv("couch_db_source") + dbName)
|
||||
fmt.Println(dbName)
|
||||
|
||||
if dbExists {
|
||||
|
||||
totalDocCount := getTotalDocumentCount(dbName)
|
||||
skip := 0
|
||||
limit := 100
|
||||
|
||||
for skip < totalDocCount {
|
||||
|
||||
var response models.BagistoModelResponse
|
||||
|
||||
url := fmt.Sprintf("%s%s/_all_docs?include_docs=true&limit=%v&skip=%v", os.Getenv("couch_db_source"), dbName, limit, skip)
|
||||
|
||||
fmt.Println(url)
|
||||
|
||||
skip += limit
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "", true)
|
||||
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &response)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
//itearate 100 row products
|
||||
for _, element := range response.Rows {
|
||||
importProduct(element.Doc, db)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fmt.Println(dbName + "+doesnt exist")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getTotalDocumentCount(db string) int {
|
||||
|
||||
var response models.DBDocCountResponse
|
||||
|
||||
url := os.Getenv("couch_db_source") + db
|
||||
|
||||
body, err := helper.SendRequest("GET", url, nil, "", true)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return 0
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return 0
|
||||
}
|
||||
|
||||
return response.DocCount
|
||||
}
|
||||
|
||||
func importProduct(product models.Product, db *gorm.DB) {
|
||||
|
||||
famAndSellerWG.Wait() //wait until attribute families and sellers are not get from mysql
|
||||
|
||||
var categories []gm.Category
|
||||
var brand gm.Brand
|
||||
|
||||
errCat := db.Find(&categories, product.Categories).Error
|
||||
|
||||
if errCat != nil {
|
||||
log.Println(errCat)
|
||||
return
|
||||
}
|
||||
|
||||
if product.Brand != "" {
|
||||
brand = gm.FindOrCreateBrand(db, product.Brand, categories)
|
||||
}
|
||||
|
||||
iproduct := gm.Product{
|
||||
Sku: product.ProductGroupID,
|
||||
Type: "simple",
|
||||
AttributeFamilyID: families[0].ID,
|
||||
Brand: brand,
|
||||
}
|
||||
|
||||
for _, element := range product.Images {
|
||||
iproduct.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element})
|
||||
}
|
||||
|
||||
if product.ColorVariantCount > 0 || len(product.SizeVariants) > 0 {
|
||||
iproduct.Type = "configurable"
|
||||
iproduct.AttributeFamilyID = families[1].ID
|
||||
}
|
||||
|
||||
err := db.Select("Sku", "Type", "AttributeFamilyID", "CreatedAt", "UpdatedAt").Create(&iproduct).Error
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
// mainProductFlat := gm.ProductFlat{}
|
||||
mainProductAttributeValues := assignAttributes(iproduct.ID, &product)
|
||||
|
||||
if iproduct.Type == "simple" {
|
||||
mainProductAttributeValues = append(mainProductAttributeValues, getPriceAttributes(iproduct.ID, &product.Price)...)
|
||||
} else if len(product.ColorVariants) > 0 || len(product.SizeVariants) > 0 {
|
||||
|
||||
var productVariants []gm.Product
|
||||
|
||||
if len(product.ColorVariants) > 0 {
|
||||
//attach super attributes color
|
||||
superAttribute := gm.ProductSuperAttribute{ProductID: iproduct.ID, AttributeID: attributesMap["color"].ID}
|
||||
db.Create(&superAttribute)
|
||||
var description string
|
||||
for _, colorVariant := range product.ColorVariants {
|
||||
weight, _ := strconv.ParseFloat(colorVariant.Weight, 64)
|
||||
|
||||
for _, desc := range colorVariant.Descriptions {
|
||||
description += "<p>" + desc.Description + "</p>"
|
||||
}
|
||||
|
||||
colorOption := gm.GetAttributeOption(db, attributesMap["color"].ID, colorVariant.Color)
|
||||
|
||||
if len(colorVariant.SizeVariants) > 0 {
|
||||
|
||||
for _, sizeVariant := range colorVariant.SizeVariants {
|
||||
|
||||
if sizeVariant.Sellable {
|
||||
sku := fmt.Sprintf("%s-%s-%d-col-size", colorVariant.ProductGroupID, colorVariant.ProductNumber, sizeVariant.ItemNumber)
|
||||
sizeOption := gm.GetAttributeOption(db, attributesMap["size"].ID, sizeVariant.AttributeValue)
|
||||
attributes := []gm.ProductAttributeValue{
|
||||
{AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["product_number"].ID, TextValue: fmt.Sprintf("%s-%d", colorVariant.ProductNumber, sizeVariant.ItemNumber), Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["name"].ID, TextValue: colorVariant.Name, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["source"].ID, TextValue: colorVariant.URLKey, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: colorVariant.FavoriteCount, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
|
||||
{
|
||||
AttributeID: attributesMap["color"].ID,
|
||||
IntegerValue: int(colorOption.ID),
|
||||
Channel: "default",
|
||||
Locale: "tm",
|
||||
}, {
|
||||
AttributeID: attributesMap["size"].ID,
|
||||
IntegerValue: int(sizeOption.ID),
|
||||
Channel: "default",
|
||||
Locale: "tm",
|
||||
},
|
||||
}
|
||||
|
||||
if sizeVariant.Price.OriginalPrice.Value > sizeVariant.Price.DiscountedPrice.Value {
|
||||
attributes = append(attributes, []gm.ProductAttributeValue{
|
||||
{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["special_price"].ID, FloatValue: sizeVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"},
|
||||
}...)
|
||||
|
||||
} else {
|
||||
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"})
|
||||
}
|
||||
|
||||
productVariant := gm.Product{
|
||||
ParentID: iproduct.ID,
|
||||
Type: "simple",
|
||||
AttributeFamily: iproduct.AttributeFamily,
|
||||
Sku: sku,
|
||||
Brand: iproduct.Brand,
|
||||
AttributeValues: attributes,
|
||||
AttributeFamilyID: iproduct.AttributeFamilyID,
|
||||
}
|
||||
|
||||
for _, element := range colorVariant.Images {
|
||||
productVariant.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element})
|
||||
}
|
||||
//todo assign images
|
||||
productVariants = append(productVariants, productVariant)
|
||||
}
|
||||
|
||||
}
|
||||
} else if colorVariant.IsSellable {
|
||||
|
||||
colorOption := gm.GetAttributeOption(db, attributesMap["color"].ID, colorVariant.Color)
|
||||
attributes := collectAttributes(&colorVariant, &colorOption)
|
||||
|
||||
if colorVariant.Price.OriginalPrice.Value > colorVariant.Price.DiscountedPrice.Value {
|
||||
attributes = append(attributes, []gm.ProductAttributeValue{
|
||||
{AttributeID: attributesMap["price"].ID, FloatValue: colorVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["special_price"].ID, FloatValue: colorVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"},
|
||||
}...)
|
||||
|
||||
} else {
|
||||
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: colorVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"})
|
||||
}
|
||||
|
||||
sku := fmt.Sprintf("%s-%s-col", iproduct.Sku, colorVariant.ProductNumber)
|
||||
|
||||
productVariant := gm.Product{
|
||||
ParentID: iproduct.ID,
|
||||
Type: "simple",
|
||||
AttributeFamily: iproduct.AttributeFamily,
|
||||
Sku: sku,
|
||||
Brand: iproduct.Brand,
|
||||
AttributeValues: attributes,
|
||||
AttributeFamilyID: iproduct.AttributeFamilyID,
|
||||
}
|
||||
|
||||
for _, element := range colorVariant.Images {
|
||||
productVariant.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element})
|
||||
}
|
||||
|
||||
productVariants = append(productVariants, productVariant)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(product.SizeVariants) > 0 {
|
||||
//attach super attributes size
|
||||
superAttribute := gm.ProductSuperAttribute{ProductID: iproduct.ID, AttributeID: attributesMap["size"].ID}
|
||||
db.Create(&superAttribute)
|
||||
|
||||
var description string
|
||||
for _, desc := range product.Descriptions {
|
||||
description += "<p>" + desc.Description + "</p>"
|
||||
}
|
||||
weight, _ := strconv.ParseFloat(product.Weight, 64)
|
||||
|
||||
for _, sizeVariant := range product.SizeVariants {
|
||||
|
||||
if sizeVariant.Sellable {
|
||||
sku := fmt.Sprintf("%s-%s-%d-size", iproduct.Sku, product.ProductNumber, sizeVariant.ItemNumber)
|
||||
sizeOption := gm.GetAttributeOption(db, attributesMap["size"].ID, sizeVariant.AttributeValue)
|
||||
|
||||
attributes := []gm.ProductAttributeValue{
|
||||
{AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["product_number"].ID, TextValue: string(sizeVariant.ItemNumber), Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["name"].ID, TextValue: product.Name, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["source"].ID, TextValue: product.URLKey, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: product.FavoriteCount, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
|
||||
{
|
||||
AttributeID: attributesMap["size"].ID,
|
||||
IntegerValue: int(sizeOption.ID),
|
||||
Channel: "default",
|
||||
Locale: "tm",
|
||||
},
|
||||
}
|
||||
|
||||
if sizeVariant.Price.OriginalPrice.Value > sizeVariant.Price.DiscountedPrice.Value {
|
||||
attributes = append(attributes, []gm.ProductAttributeValue{
|
||||
{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["special_price"].ID, FloatValue: sizeVariant.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"},
|
||||
}...)
|
||||
|
||||
} else {
|
||||
attributes = append(attributes, gm.ProductAttributeValue{AttributeID: attributesMap["price"].ID, FloatValue: sizeVariant.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"})
|
||||
}
|
||||
|
||||
productVariant := gm.Product{
|
||||
ParentID: iproduct.ID,
|
||||
Type: "simple",
|
||||
AttributeFamily: iproduct.AttributeFamily,
|
||||
Sku: sku,
|
||||
Brand: iproduct.Brand,
|
||||
AttributeValues: attributes,
|
||||
AttributeFamilyID: iproduct.AttributeFamilyID,
|
||||
}
|
||||
|
||||
// for _, element := range colorVariant.Images {
|
||||
// productVariant.Images = append(iproduct.Images, gm.ProductImage{Type: "cdn", Path: element})
|
||||
// }
|
||||
|
||||
productVariants = append(productVariants, productVariant)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
db.Create(&productVariants) //todo assign all with categories
|
||||
}
|
||||
|
||||
db.Model(&iproduct).Association("Categories").Append(categories)
|
||||
var err2 = db.Create(&mainProductAttributeValues).Error
|
||||
|
||||
if err2 != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
// panic("'hassiktir'")
|
||||
//todo gorm transactions
|
||||
//todo Attributes source, cinsiyet
|
||||
|
||||
// if len(product.Attributes) > 0 {
|
||||
// iproduct.AttrbuteFamily = getAttributeFamily()
|
||||
// }
|
||||
|
||||
//todo seller product, meta keys
|
||||
// if($data['vendor'] && $seller = $this->vendorRepository->findOneByField('url',$data['vendor'])){
|
||||
// $this->createSellerProduct($parentProduct, $seller->id);
|
||||
// }
|
||||
|
||||
if product.Vendor != "" {
|
||||
for i := range sellers {
|
||||
if sellers[i].Url == product.Vendor {
|
||||
createSellerProduct()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
func createSellerProduct() {
|
||||
/*
|
||||
Event::dispatch('marketplace.catalog.product.create.before');
|
||||
|
||||
$sellerProduct = $this->vendorProductRepository->create([
|
||||
'marketplace_seller_id' => $seller_id,
|
||||
'is_approved' => 1,
|
||||
'condition' => 'new',
|
||||
'description' => 'scraped product',
|
||||
'is_owner' => 1,
|
||||
'product_id' => $product->id,
|
||||
]);
|
||||
|
||||
foreach ($sellerProduct->product->variants as $baseVariant) {
|
||||
$this->vendorProductRepository->create([
|
||||
'parent_id' => $sellerProduct->id,
|
||||
'product_id' => $baseVariant->id,
|
||||
'is_owner' => 1,
|
||||
'marketplace_seller_id' => $seller_id,
|
||||
'is_approved' => 1,
|
||||
'condition' => 'new',
|
||||
]);
|
||||
}
|
||||
|
||||
Event::dispatch('marketplace.catalog.product.create.after', $sellerProduct);
|
||||
|
||||
return $sellerProduct;
|
||||
*/
|
||||
}
|
||||
func collectAttributes(variant *models.Product, option *gm.AttributeOption) []gm.ProductAttributeValue {
|
||||
sku := fmt.Sprintf("%s-%s", variant.ProductGroupID, variant.ProductNumber)
|
||||
weight, _ := strconv.ParseFloat(variant.Weight, 64)
|
||||
var description string
|
||||
|
||||
for _, desc := range variant.Descriptions {
|
||||
description += "<p>" + desc.Description + "</p>"
|
||||
}
|
||||
|
||||
return []gm.ProductAttributeValue{
|
||||
{AttributeID: attributesMap["sku"].ID, TextValue: sku, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["product_number"].ID, TextValue: variant.ProductNumber, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["name"].ID, TextValue: variant.Name, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["source"].ID, TextValue: variant.URLKey, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["url_key"].ID, TextValue: sku, Channel: "default", Locale: "tm"},
|
||||
// {AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: variant.FavoriteCount, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
|
||||
{AttributeID: option.AttributeID, IntegerValue: int(option.ID), Channel: "default", Locale: "tm"},
|
||||
}
|
||||
}
|
||||
|
||||
func assignAttributes(productID uint, data *models.Product) []gm.ProductAttributeValue {
|
||||
|
||||
weight, _ := strconv.ParseFloat(data.Weight, 64)
|
||||
|
||||
var description string
|
||||
|
||||
for _, desc := range data.Descriptions {
|
||||
description += "<p>" + desc.Description + "</p>"
|
||||
}
|
||||
//$desc = implode(array_map(fn($value): string => '<p>' . $value['description'] . '</p>', $data['descriptions']));
|
||||
|
||||
var productAttributeValues = []gm.ProductAttributeValue{
|
||||
{ProductID: productID, AttributeID: attributesMap["sku"].ID, TextValue: data.ProductGroupID, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["product_number"].ID, TextValue: data.ProductNumber, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["name"].ID, TextValue: data.Name, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["weight"].ID, FloatValue: weight, Channel: "default", Locale: "tm"},
|
||||
// {ProductID: productID, AttributeID: attributesMap["source"].ID, TextValue: data.URLKey, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["status"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["visible_individually"].ID, BooleanValue: true, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["url_key"].ID, TextValue: data.ProductGroupID, Channel: "default", Locale: "tm"},
|
||||
// {ProductID: productID, AttributeID: attributesMap["short_description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["description"].ID, TextValue: description, Channel: "default", Locale: "tm"},
|
||||
// {ProductID: productID, AttributeID: attributesMap["favoritesCount"].ID, IntegerValue: data.FavoriteCount, Channel: "default", Locale: "tm"},
|
||||
}
|
||||
|
||||
// if data.Price.OriginalPrice.Value > data.Price.DiscountedPrice.Value {
|
||||
// productAttributeValues = append(productAttributeValues, []gm.ProductAttributeValue{
|
||||
// {ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: data.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"},
|
||||
// {ProductID: productID, AttributeID: attributesMap["special_price"].ID, FloatValue: data.Price.DiscountedPrice.Value, Channel: "default", Locale: "tm"},
|
||||
// }...)
|
||||
|
||||
// } else {
|
||||
// productAttributeValues = append(productAttributeValues, gm.ProductAttributeValue{ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: data.Price.OriginalPrice.Value, Channel: "default", Locale: "tm"})
|
||||
|
||||
// }
|
||||
|
||||
return productAttributeValues
|
||||
}
|
||||
|
||||
func getPriceAttributes(productID uint, price *models.Price) []gm.ProductAttributeValue {
|
||||
var productAttributeValues []gm.ProductAttributeValue
|
||||
if price.OriginalPrice.Value > price.DiscountedPrice.Value {
|
||||
productAttributeValues = []gm.ProductAttributeValue{
|
||||
{ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: price.OriginalPrice.Value, Channel: "default", Locale: "tm"},
|
||||
{ProductID: productID, AttributeID: attributesMap["special_price"].ID, FloatValue: price.DiscountedPrice.Value, Channel: "default", Locale: "tm"},
|
||||
}
|
||||
|
||||
} else {
|
||||
productAttributeValues = []gm.ProductAttributeValue{{ProductID: productID, AttributeID: attributesMap["price"].ID, FloatValue: price.OriginalPrice.Value, Channel: "default", Locale: "tm"}}
|
||||
|
||||
}
|
||||
|
||||
return productAttributeValues
|
||||
}
|
||||
|
||||
func getAttributeFamily() gm.AttributeFamily {
|
||||
|
||||
return families[0]
|
||||
}
|
||||
Loading…
Reference in New Issue