我无法在 Golang Colly 的 JSON 数组中并排打印数据

Question

我正在用亚马逊的 golang colly 拍照，我想将这些图片 JSON 放入一个数组中（每个产品图片只有 1 个数组）。我抓取了我需要的图片，只是 JSON 文件有问题。非常感谢您。

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    AllInfos := make([]Info, 0)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    Counter := 0
    var info Info
    var theArray [10]string
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/I/" && imgclass == "" {
            theArray[Counter] = imgsrc
            fmt.Printf("The Array %d %v", Counter, theArray[Counter]+"\n")
            Counter = Counter + 1
            co.Visit(e.Request.AbsoluteURL(imgsrc))

            info = Info{
                Images: []string{
                    theArray[0],
                    theArray[1],
                    theArray[2],
                    theArray[3],
                    theArray[4],
                    theArray[5],
                    theArray[6],
                },
            }
            AllInfos = append(AllInfos, info)
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(AllInfos)

    writeJson(AllInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data []Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

我的代码的 JSON 结果；

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
   ""
  ]
 }
]

我需要 JSON 这样的结果；

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
  ]
 }
]

Answer 1

我很难理解你想做什么，但这是我想到的：

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    allInfos := new(Info)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
            fmt.Printf("The Array %d %v", len(allInfos.Images), imgsrc+"\n")
            allInfos.Images = append(allInfos.Images, imgsrc)
            co.Visit(e.Request.AbsoluteURL(imgsrc))
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(allInfos)

    writeJson(allInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data *Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

正如@Flimzy 所建议的，我仅将新的 imagesrc 附加到信息结构的 since 实例。看来您的预期输出是包含所有图像的长度为 1 的切片。

我还使用了 strings.HasPrefix 来清理您的切片检查。这是等效的，但我认为它更简洁，所以这是一个可选的更改。

如果您还有其他问题，请告诉我。

编辑，如果您需要继续使用切片，则类似以下内容可能会起作用。从您的问题中很难判断您何时需要清除新信息：

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    allInfos := make([]Info, 1)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
            fmt.Printf("The Array %d %v", len(allInfos[0].Images), imgsrc+"\n")
            allInfos[0].Images = append(allInfos[0].Images, imgsrc)
            co.Visit(e.Request.AbsoluteURL(imgsrc))
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(allInfos)

    writeJson(allInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data []Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

我无法在 Golang Colly 的 JSON 数组中并排打印数据

I cannot print data side by side in JSON array in Golang Colly

json

go

go-colly