Merge branch 'dev' into rss-feed-conditional-requests

This commit is contained in:
Svilen Markov
2025-04-29 10:39:19 +01:00
committed by GitHub
66 changed files with 4207 additions and 2553 deletions
+16 -6
View File
@@ -24,6 +24,9 @@ var (
const defaultClientTimeout = 5 * time.Second
var defaultHTTPClient = &http.Client{
Transport: &http.Transport{
MaxIdleConnsPerHost: 10,
},
Timeout: defaultClientTimeout,
}
@@ -41,13 +44,17 @@ type requestDoer interface {
var glanceUserAgentString = "Glance/" + buildVersion + " +https://github.com/glanceapp/glance"
var userAgentPersistentVersion atomic.Int32
func setBrowserUserAgentHeader(request *http.Request) {
func getBrowserUserAgentHeader() string {
if rand.IntN(2000) == 0 {
userAgentPersistentVersion.Store(rand.Int32N(5))
}
version := strconv.Itoa(130 + int(userAgentPersistentVersion.Load()))
request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:"+version+".0) Gecko/20100101 Firefox/"+version+".0")
return "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:" + version + ".0) Gecko/20100101 Firefox/" + version + ".0"
}
func setBrowserUserAgentHeader(request *http.Request) {
request.Header.Set("User-Agent", getBrowserUserAgentHeader())
}
func decodeJsonFromRequest[T any](client requestDoer, request *http.Request) (T, error) {
@@ -68,7 +75,7 @@ func decodeJsonFromRequest[T any](client requestDoer, request *http.Request) (T,
truncatedBody, _ := limitStringLength(string(body), 256)
return result, fmt.Errorf(
"unexpected status code %d for %s, response: %s",
"unexpected status code %d from %s, response: %s",
response.StatusCode,
request.URL,
truncatedBody,
@@ -148,10 +155,8 @@ const defaultNumWorkers = 10
func (job *workerPoolJob[I, O]) withWorkers(workers int) *workerPoolJob[I, O] {
if workers == 0 {
job.workers = defaultNumWorkers
} else if workers > len(job.data) {
job.workers = len(job.data)
} else {
job.workers = workers
job.workers = min(workers, len(job.data))
}
return job
@@ -182,6 +187,11 @@ func workerPoolDo[I any, O any](job *workerPoolJob[I, O]) ([]O, []error, error)
return results, errs, nil
}
if len(job.data) == 1 {
results[0], errs[0] = job.task(job.data[0])
return results, errs, nil
}
tasksQueue := make(chan *workerPoolTask[I, O])
resultsQueue := make(chan *workerPoolTask[I, O])