From afc3d33009ce8d82d3065c3d4cee595e74d11132 Mon Sep 17 00:00:00 2001
From: "Erik Winter (aider)" <e@ewintr.nl>
Date: Mon, 25 Nov 2024 10:26:56 +0100
Subject: [PATCH] feat: Add browser-like User-Agent string to HTTP requests

---
 linkchecker.go | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/linkchecker.go b/linkchecker.go
index 7a7c6da..dfc06d9 100644
--- a/linkchecker.go
+++ b/linkchecker.go
@@ -1,5 +1,7 @@
 package main
 
+const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
+
 import (
 	"fmt"
 	"net/http"
@@ -120,7 +122,13 @@ func (lc *LinkChecker) checkLinksRecursive(pageURL string, brokenLinks []BrokenL
 }
 
 func (lc *LinkChecker) getLinks(pageURL string) ([]string, error) {
-	resp, err := lc.client.Get(pageURL)
+	req, err := http.NewRequest("GET", pageURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	
+	resp, err := lc.client.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -169,13 +177,19 @@ func (lc *LinkChecker) getLinks(pageURL string) ([]string, error) {
 }
 
 func (lc *LinkChecker) isLinkValid(link string) (int, *RedirectInfo, error) {
+	req, err := http.NewRequest("GET", link, nil)
+	if err != nil {
+		return 0, nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return http.ErrUseLastResponse
 		},
 	}
 
-	resp, err := client.Get(link)
+	resp, err := client.Do(req)
 	if err != nil {
 		return 0, nil, err
 	}