ewintr
/
linkchecker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Add browser-like User-Agent string to HTTP requests

This commit is contained in:
Erik Winter (aider) 2024-11-25 10:26:56 +01:00
parent d2c064abda
commit afc3d33009
1 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
linkchecker.go
View File

@ -1,5 +1,7 @@
package main
const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
import (
"fmt"
"net/http"
@ -120,7 +122,13 @@ func (lc *LinkChecker) checkLinksRecursive(pageURL string, brokenLinks []BrokenL
}
func (lc *LinkChecker) getLinks(pageURL string) ([]string, error) {
resp, err := lc.client.Get(pageURL)
req, err := http.NewRequest("GET", pageURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
resp, err := lc.client.Do(req)
if err != nil {
return nil, err
}
@ -169,13 +177,19 @@ func (lc *LinkChecker) getLinks(pageURL string) ([]string, error) {
}
func (lc *LinkChecker) isLinkValid(link string) (int, *RedirectInfo, error) {
req, err := http.NewRequest("GET", link, nil)
if err != nil {
return 0, nil, err
}
req.Header.Set("User-Agent", userAgent)
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
resp, err := client.Get(link)
resp, err := client.Do(req)
if err != nil {
return 0, nil, err
}