.gitignore

@@ -1,2 +0,0 @@
-.aider*
-.env

README.md

@@ -1,64 +0,0 @@
-# Link Checker
-
-A recursive link checker that crawls websites to find broken links and redirects. It helps maintain website health by identifying:
-
-- Broken links (HTTP 4xx, 5xx status codes)
-- Network/DNS errors
-- HTTP redirects (3xx status codes)
-
-## Features
-
-- Recursive crawling of websites
-- Handles both absolute and relative URLs
-- Detects and reports HTTP redirects
-- Shows progress during scanning
-- Normalizes URLs for consistent checking
-- Stays within the same domain
-- Detailed reporting of issues found
-
-## Installation
-
-Make sure you have Go installed (version 1.16 or later), then run:
-
-```bash
-go install forgejo.ewintr.nl/ewintr/linkchecker@latest
-```
-
-## Usage
-
-Run the link checker by providing a starting URL:
-
-```bash
-linkchecker -url="https://example.com"
-```
-
-The tool will:
-1. Crawl all pages on the same domain
-2. Check all links found (both internal and external)
-3. Display progress during the scan
-4. Generate a report showing:
-   - Total pages checked
-   - List of redirected links
-   - List of broken links
-   - Summary statistics
-
-## Example Output
-
-```
-Checking page 1: https://example.com
-Checking page 2: https://example.com/about
-...
-
-Total pages checked: 15
-
-Redirects found:
-- http://example.com/old-page (Redirect 301 -> https://example.com/new-page)
-- http://example.com/blog (Redirect 302 -> https://blog.example.com)
-
-Broken links found:
-- https://example.com/missing-page (Status: 404)
-- https://example.com/server-error (Status: 500)
-- https://external-site.com/broken (Error: connection refused)
-
-Total issues: 5 (2 redirects, 3 broken)
-```

go.mod

@@ -1,5 +0,0 @@
-module go-mod.ewintr.nl/linkchecker
-
-go 1.23.3
-
-require golang.org/x/net v0.31.0

go.sum

@@ -1,2 +0,0 @@
-golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
-golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=

linkchecker.go

@@ -9,88 +9,34 @@ import (
 	"golang.org/x/net/html"
 )
 
-const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
-
 type LinkChecker struct {
-	client       *http.Client
+	client *http.Client
-	visited      map[string]bool
-	pagesChecked int
 }
 
 func NewLinkChecker() *LinkChecker {
 	return &LinkChecker{
-		client:       &http.Client{},
+		client: &http.Client{},
-		visited:      make(map[string]bool),
-		pagesChecked: 0,
 	}
 }
 
-type RedirectInfo struct {
-	FromURL string
-	ToURL   string
-}
-
 type BrokenLink struct {
 	URL        string
 	StatusCode int
 	Error      string
-	Redirect   *RedirectInfo
-}
-
-func (lc *LinkChecker) normalizeURL(rawURL string) string {
-	u, err := url.Parse(rawURL)
-	if err != nil {
-		return rawURL
-	}
-
-	// Remove fragment
-	u.Fragment = ""
-	
-	// Ensure consistent trailing slash handling
-	u.Path = strings.TrimSuffix(u.Path, "/")
-	if u.Path == "" {
-		u.Path = "/"
-	}
-
-	return u.String()
-}
-
-func (lc *LinkChecker) isSameDomain(baseURL, link string) bool {
-	base, err := url.Parse(baseURL)
-	if err != nil {
-		return false
-	}
-	
-	target, err := url.Parse(link)
-	if err != nil {
-		return false
-	}
-	
-	return base.Host == target.Host
 }
 
 func (lc *LinkChecker) CheckLinks(baseURL string) ([]BrokenLink, error) {
-	return lc.checkLinksRecursive(baseURL, make([]BrokenLink, 0))
+	// Get all links from the page
-}
+	links, err := lc.getLinks(baseURL)
-
-func (lc *LinkChecker) checkLinksRecursive(pageURL string, brokenLinks []BrokenLink) ([]BrokenLink, error) {
-	normalizedURL := lc.normalizeURL(pageURL)
-	if lc.visited[normalizedURL] {
-		return brokenLinks, nil
-	}
-	lc.visited[normalizedURL] = true
-	lc.pagesChecked++
-	
-	fmt.Printf("Checking page %d: %s\n", lc.pagesChecked, pageURL)
-	
-	links, err := lc.getLinks(pageURL)
 	if err != nil {
 		return nil, fmt.Errorf("error getting links: %w", err)
 	}
 
+	var brokenLinks []BrokenLink
+
+	// Check each link
 	for _, link := range links {
-		status, redirect, err := lc.isLinkValid(link)
+		if status, err := lc.isLinkValid(link); status >= 400 || err != nil {
-		if status >= 400 || err != nil {
 			broken := BrokenLink{URL: link}
 			if err != nil {
 				broken.Error = err.Error()
@@ -98,23 +44,6 @@ func (lc *LinkChecker) checkLinksRecursive(pageURL string, brokenLinks []BrokenL
 				broken.StatusCode = status
 			}
 			brokenLinks = append(brokenLinks, broken)
-		} else if redirect != nil {
-			broken := BrokenLink{
-				URL:      link,
-				StatusCode: status,
-				Redirect: redirect,
-			}
-			brokenLinks = append(brokenLinks, broken)
-		}
-		
-		// Recursively check links from the same domain
-		normalizedLink := lc.normalizeURL(link)
-		if lc.isSameDomain(pageURL, link) && !lc.visited[normalizedLink] {
-			recursiveLinks, err := lc.checkLinksRecursive(link, brokenLinks)
-			if err != nil {
-				continue // Skip this page if there's an error, but continue checking others
-			}
-			brokenLinks = recursiveLinks
 		}
 	}
 
@@ -122,13 +51,7 @@ func (lc *LinkChecker) checkLinksRecursive(pageURL string, brokenLinks []BrokenL
 }
 
 func (lc *LinkChecker) getLinks(pageURL string) ([]string, error) {
-	req, err := http.NewRequest("GET", pageURL, nil)
+	resp, err := lc.client.Get(pageURL)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("User-Agent", userAgent)
-	
-	resp, err := lc.client.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -176,41 +99,12 @@ func (lc *LinkChecker) getLinks(pageURL string) ([]string, error) {
 	return links, nil
 }
 
-func (lc *LinkChecker) isLinkValid(link string) (int, *RedirectInfo, error) {
+func (lc *LinkChecker) isLinkValid(link string) (int, error) {
-	req, err := http.NewRequest("GET", link, nil)
+	resp, err := lc.client.Get(link)
 	if err != nil {
-		return 0, nil, err
+		return 0, err
-	}
-	req.Header.Set("User-Agent", userAgent)
-
-	client := &http.Client{
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return 0, nil, err
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+	return resp.StatusCode, nil
-		location := resp.Header.Get("Location")
-		if location != "" {
-			redirectURL := location
-			if !strings.HasPrefix(location, "http") {
-				baseURL, _ := url.Parse(link)
-				if relative, err := baseURL.Parse(location); err == nil {
-					redirectURL = relative.String()
-				}
-			}
-			return resp.StatusCode, &RedirectInfo{
-				FromURL: link,
-				ToURL:   redirectURL,
-			}, nil
-		}
-	}
-
-	return resp.StatusCode, nil, nil
 }

main.go

@@ -20,44 +20,17 @@ func main() {
 		log.Fatal(err)
 	}
 
-	fmt.Printf("\nTotal pages checked: %d\n\n", checker.pagesChecked)
-	
 	if len(brokenLinks) == 0 {
-		fmt.Println("No issues found!")
+		fmt.Println("No broken links found!")
 		return
 	}
 
-	// First list redirects
+	fmt.Println("Found broken links:")
-	var redirectCount int
 	for _, link := range brokenLinks {
-		if link.Redirect != nil {
+		if link.Error != "" {
-			if redirectCount == 0 {
+			fmt.Printf("- %s (Error: %s)\n", link.URL, link.Error)
-				fmt.Println("Redirects found:")
+		} else {
-			}
+			fmt.Printf("- %s (Status: %d)\n", link.URL, link.StatusCode)
-			fmt.Printf("- %s (Redirect %d -> %s)\n", link.URL, link.StatusCode, link.Redirect.ToURL)
-			redirectCount++
 		}
 	}
-	if redirectCount > 0 {
-		fmt.Println()
-	}
-
-	// Then list broken links
-	var brokenCount int
-	for _, link := range brokenLinks {
-		if link.Redirect == nil {
-			if brokenCount == 0 {
-				fmt.Println("Broken links found:")
-			}
-			if link.Error != "" {
-				fmt.Printf("- %s (Error: %s)\n", link.URL, link.Error)
-			} else {
-				fmt.Printf("- %s (Status: %d)\n", link.URL, link.StatusCode)
-			}
-			brokenCount++
-		}
-	}
-
-	fmt.Printf("\nTotal issues: %d (%d redirects, %d broken)\n", 
-		len(brokenLinks), redirectCount, brokenCount)
 }