utils

package

v0.0.0-...-1910856 Latest Latest Go to latest Published: Jul 14, 2022 License: MIT Imports: 5 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/Los-had/qmts-crawler

Links

Open Source Insights

Documentation ¶

Constants ¶

This section is empty.

Variables ¶

View Source

var BraveApiURL string = "https://search.brave.com/api/suggest?q="

View Source

var GoogleApiURL string = "https://suggestqueries.google.com/complete/search?client=toolbar&q="

View Source

var UserAgent = "QMTSbot/0.1.1"

View Source

var WikipediaApiURL string = "https://en.wikipedia.org/w/api.php?action=opensearch&limit=5&namespace=0&format=json&search="

Functions ¶

func CheckURL ¶

func CheckURL(rawURL string) bool

Check if is a valid URL

func CreateHash ¶

func CreateHash(link string) string

Generate a unique hash by passing a url

func MakeURLS ¶

func MakeURLS(url string) []string

Generate urls

Types ¶

type Image ¶

type Image struct {
	URL  string `json:"url"`
	Alt  string `json:"alt"`
	Host string `json:"host"`
}

type PageData ¶

type PageData struct {
	URL         string `json:"url"`
	Title       string `json:"title"`
	Description string `json:"description"`
}

func GetAboutPage ¶

func GetAboutPage(site string) PageData

Get the about page

func GetContactsPage ¶

func GetContactsPage(site string) PageData

Get the contacts page

func GetDownloadPage ¶

func GetDownloadPage(site string) PageData

Get the download page

func GetFAQtPage ¶

func GetFAQtPage(site string) PageData

Get the FAQ page

type Result ¶

type Result struct {
	Favicon     string   `json:"favicon"`
	URL         string   `json:"url"`
	Lang        string   `json:"lang"`
	Title       string   `json:"title"`
	Description string   `json:"description"`
	Keywords    []string `json:"keywords"`
	SitePages   SiteData `json:"pages"`
	Images      []Image  `json:"images"`
	Visited     bool     `json:"visited"`
	VisitedTime string   `json:"time"`
	Hash        string   `json:"hash"`
}

func ParseResult ¶

func ParseResult(r Result) Result

Parse the Result struct

type Seed ¶

type Seed struct {
	Host   string `json:"host"`
	Scheme string `json:"scheme"`
	Params string `json:"params"`
	Port   string `json:"port"`
}

func GetSeedInfo ¶

func GetSeedInfo(seed string) Seed

Get information of an especific domain

type SiteData ¶

type SiteData struct {
	AboutPage    PageData `json:"about"`
	ContactsPage PageData `json:"contacts"`
	FAQPage      PageData `json:"FAQ"`
	DownloadPage PageData `json:"download"`
}

type Suggestion ¶

type Suggestion struct {
	Text string `json:"text"`
	From string `json:"from"`
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL