forgejo/modules/cache/cache.go

186 lines
3.8 KiB
Go
Raw Normal View History

// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"errors"
"fmt"
"strconv"
"time"
"forgejo.org/modules/setting"
mc "code.forgejo.org/go-chi/cache"
_ "code.forgejo.org/go-chi/cache/memcache" // memcache plugin for cache
)
var conn mc.Cache
func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
Move macaron to chi (#14293) Use [chi](https://github.com/go-chi/chi) instead of the forked [macaron](https://gitea.com/macaron/macaron). Since macaron and chi have conflicts with session share, this big PR becomes a have-to thing. According my previous idea, we can replace macaron step by step but I'm wrong. :( Below is a list of big changes on this PR. - [x] Define `context.ResponseWriter` interface with an implementation `context.Response`. - [x] Use chi instead of macaron, and also a customize `Route` to wrap chi so that the router usage is similar as before. - [x] Create different routers for `web`, `api`, `internal` and `install` so that the codes will be more clear and no magic . - [x] Use https://github.com/unrolled/render instead of macaron's internal render - [x] Use https://github.com/NYTimes/gziphandler instead of https://gitea.com/macaron/gzip - [x] Use https://gitea.com/go-chi/session which is a modified version of https://gitea.com/macaron/session and removed `nodb` support since it will not be maintained. **BREAK** - [x] Use https://gitea.com/go-chi/captcha which is a modified version of https://gitea.com/macaron/captcha - [x] Use https://gitea.com/go-chi/cache which is a modified version of https://gitea.com/macaron/cache - [x] Use https://gitea.com/go-chi/binding which is a modified version of https://gitea.com/macaron/binding - [x] Use https://github.com/go-chi/cors instead of https://gitea.com/macaron/cors - [x] Dropped https://gitea.com/macaron/i18n and make a new one in `code.gitea.io/gitea/modules/translation` - [x] Move validation form structs from `code.gitea.io/gitea/modules/auth` to `code.gitea.io/gitea/modules/forms` to avoid dependency cycle. - [x] Removed macaron log service because it's not need any more. **BREAK** - [x] All form structs have to be get by `web.GetForm(ctx)` in the route function but not as a function parameter on routes definition. - [x] Move Git HTTP protocol implementation to use routers directly. - [x] Fix the problem that chi routes don't support trailing slash but macaron did. - [x] `/api/v1/swagger` now will be redirect to `/api/swagger` but not render directly so that `APIContext` will not create a html render. Notices: - Chi router don't support request with trailing slash - Integration test `TestUserHeatmap` maybe mysql version related. It's failed on my macOS(mysql 5.7.29 installed via brew) but succeed on CI. Co-authored-by: 6543 <6543@obermui.de>
2021-01-26 10:36:53 -05:00
return mc.NewCacher(mc.Options{
Adapter: cacheConfig.Adapter,
AdapterConfig: cacheConfig.Conn,
Interval: cacheConfig.Interval,
})
}
// Init start cache service
func Init() error {
var err error
if conn == nil {
if conn, err = newCache(setting.CacheService.Cache); err != nil {
return err
}
if err = conn.Ping(); err != nil {
2021-12-05 11:24:57 -05:00
return err
}
}
return err
}
const (
testCacheKey = "DefaultCache.TestKey"
SlowCacheThreshold = 100 * time.Microsecond
)
func Test() (time.Duration, error) {
if conn == nil {
return 0, errors.New("default cache not initialized")
}
testData := fmt.Sprintf("%x", make([]byte, 500))
start := time.Now()
if err := conn.Delete(testCacheKey); err != nil {
return 0, fmt.Errorf("expect cache to delete data based on key if exist but got: %w", err)
}
if err := conn.Put(testCacheKey, testData, 10); err != nil {
return 0, fmt.Errorf("expect cache to store data but got: %w", err)
}
testVal := conn.Get(testCacheKey)
if testVal == nil {
return 0, errors.New("expect cache hit but got none")
}
if testVal != testData {
return 0, errors.New("expect cache to return same value as stored but got other")
}
return time.Since(start), nil
}
Use native git variants by default with go-git variants as build tag (#13673) * Move last commit cache back into modules/git Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove go-git from the interface for last commit cache Signed-off-by: Andrew Thornton <art27@cantab.net> * move cacheref to last_commit_cache Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove go-git from routers/private/hook Signed-off-by: Andrew Thornton <art27@cantab.net> * Move FindLFSFiles to pipeline Signed-off-by: Andrew Thornton <art27@cantab.net> * Make no-go-git variants Signed-off-by: Andrew Thornton <art27@cantab.net> * Submodule RefID Signed-off-by: Andrew Thornton <art27@cantab.net> * fix issue with GetCommitsInfo Signed-off-by: Andrew Thornton <art27@cantab.net> * fix GetLastCommitForPaths Signed-off-by: Andrew Thornton <art27@cantab.net> * Improve efficiency Signed-off-by: Andrew Thornton <art27@cantab.net> * More efficiency Signed-off-by: Andrew Thornton <art27@cantab.net> * even faster Signed-off-by: Andrew Thornton <art27@cantab.net> * Reduce duplication * As per @lunny Signed-off-by: Andrew Thornton <art27@cantab.net> * attempt to fix drone Signed-off-by: Andrew Thornton <art27@cantab.net> * fix test-tags Signed-off-by: Andrew Thornton <art27@cantab.net> * default to use no-go-git variants and add gogit build tag Signed-off-by: Andrew Thornton <art27@cantab.net> * placate lint Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @6543 Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
2020-12-17 09:00:47 -05:00
// GetCache returns the currently configured cache
func GetCache() mc.Cache {
Use native git variants by default with go-git variants as build tag (#13673) * Move last commit cache back into modules/git Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove go-git from the interface for last commit cache Signed-off-by: Andrew Thornton <art27@cantab.net> * move cacheref to last_commit_cache Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove go-git from routers/private/hook Signed-off-by: Andrew Thornton <art27@cantab.net> * Move FindLFSFiles to pipeline Signed-off-by: Andrew Thornton <art27@cantab.net> * Make no-go-git variants Signed-off-by: Andrew Thornton <art27@cantab.net> * Submodule RefID Signed-off-by: Andrew Thornton <art27@cantab.net> * fix issue with GetCommitsInfo Signed-off-by: Andrew Thornton <art27@cantab.net> * fix GetLastCommitForPaths Signed-off-by: Andrew Thornton <art27@cantab.net> * Improve efficiency Signed-off-by: Andrew Thornton <art27@cantab.net> * More efficiency Signed-off-by: Andrew Thornton <art27@cantab.net> * even faster Signed-off-by: Andrew Thornton <art27@cantab.net> * Reduce duplication * As per @lunny Signed-off-by: Andrew Thornton <art27@cantab.net> * attempt to fix drone Signed-off-by: Andrew Thornton <art27@cantab.net> * fix test-tags Signed-off-by: Andrew Thornton <art27@cantab.net> * default to use no-go-git variants and add gogit build tag Signed-off-by: Andrew Thornton <art27@cantab.net> * placate lint Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @6543 Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
2020-12-17 09:00:47 -05:00
return conn
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
fix: reduce deadlocks merging PRs by using caching for repo issue count stats (#9922) The `repository` table has quite a few "count of related objects" fields on it, including the number of issues, closed issues, pull requests, and closed pull requests. These fields specifically will cause deadlocks during concurrent PR merges as documented in #9785. These fields are not used in database queries. In order to eliminate the deadlock possibility on them, I've moved them to be calculated on-demand with caching, with the cache being invalidated in the same places that the recalc used to be triggered. I've supplemented the already in-place automated testing with manual testing performing simple close & reopen of issues & PRs, and the counts which are used in the tabs at the top of the repo page are updated correctly as expected. Near future work: - Similar change can probably be performed to fix #9846 - Last known deadlock identified from #9785; I'm hoping to incorporate the synthetic deadlock test in a near future PR to prevent regressions ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - Tests were already in-place covering these fields; they've been adjusted from using the fields to the new accessor methods. - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [x] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title. <!--start release-notes-assistant--> ## Release notes <!--URL:https://codeberg.org/forgejo/forgejo--> - Bug fixes - [PR](https://codeberg.org/forgejo/forgejo/pulls/9922): <!--number 9922 --><!--line 0 --><!--description cmVkdWNlIGRlYWRsb2NrcyBtZXJnaW5nIFBScyBieSB1c2luZyBjYWNoaW5nIGZvciByZXBvIGlzc3VlIGNvdW50IHN0YXRz-->reduce deadlocks merging PRs by using caching for repo issue count stats<!--description--> <!--end release-notes-assistant--> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/9922 Reviewed-by: Gusted <gusted@noreply.codeberg.org> Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
2025-10-31 18:50:05 -04:00
if conn == nil || setting.CacheService.TTL <= 0 {
return getFunc()
}
cached := conn.Get(key)
if cached == nil {
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
if value, ok := cached.(string); ok {
return value, nil
}
if stringer, ok := cached.(fmt.Stringer); ok {
return stringer.String(), nil
}
return fmt.Sprintf("%s", cached), nil
}
// GetInt returns key value from cache with callback when no key exists in cache
func GetInt(key string, getFunc func() (int, error)) (int, error) {
fix: reduce deadlocks merging PRs by using caching for repo issue count stats (#9922) The `repository` table has quite a few "count of related objects" fields on it, including the number of issues, closed issues, pull requests, and closed pull requests. These fields specifically will cause deadlocks during concurrent PR merges as documented in #9785. These fields are not used in database queries. In order to eliminate the deadlock possibility on them, I've moved them to be calculated on-demand with caching, with the cache being invalidated in the same places that the recalc used to be triggered. I've supplemented the already in-place automated testing with manual testing performing simple close & reopen of issues & PRs, and the counts which are used in the tabs at the top of the repo page are updated correctly as expected. Near future work: - Similar change can probably be performed to fix #9846 - Last known deadlock identified from #9785; I'm hoping to incorporate the synthetic deadlock test in a near future PR to prevent regressions ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - Tests were already in-place covering these fields; they've been adjusted from using the fields to the new accessor methods. - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [x] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title. <!--start release-notes-assistant--> ## Release notes <!--URL:https://codeberg.org/forgejo/forgejo--> - Bug fixes - [PR](https://codeberg.org/forgejo/forgejo/pulls/9922): <!--number 9922 --><!--line 0 --><!--description cmVkdWNlIGRlYWRsb2NrcyBtZXJnaW5nIFBScyBieSB1c2luZyBjYWNoaW5nIGZvciByZXBvIGlzc3VlIGNvdW50IHN0YXRz-->reduce deadlocks merging PRs by using caching for repo issue count stats<!--description--> <!--end release-notes-assistant--> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/9922 Reviewed-by: Gusted <gusted@noreply.codeberg.org> Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
2025-10-31 18:50:05 -04:00
if conn == nil || setting.CacheService.TTL <= 0 {
return getFunc()
}
cached := conn.Get(key)
if cached == nil {
value, err := getFunc()
2019-06-12 15:41:28 -04:00
if err != nil {
return value, err
2019-06-12 15:41:28 -04:00
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
switch v := cached.(type) {
case int:
return v, nil
case string:
value, err := strconv.Atoi(v)
if err != nil {
return 0, err
}
return value, nil
default:
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
}
// GetInt64 returns key value from cache with callback when no key exists in cache
func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
fix: reduce deadlocks merging PRs by using caching for repo issue count stats (#9922) The `repository` table has quite a few "count of related objects" fields on it, including the number of issues, closed issues, pull requests, and closed pull requests. These fields specifically will cause deadlocks during concurrent PR merges as documented in #9785. These fields are not used in database queries. In order to eliminate the deadlock possibility on them, I've moved them to be calculated on-demand with caching, with the cache being invalidated in the same places that the recalc used to be triggered. I've supplemented the already in-place automated testing with manual testing performing simple close & reopen of issues & PRs, and the counts which are used in the tabs at the top of the repo page are updated correctly as expected. Near future work: - Similar change can probably be performed to fix #9846 - Last known deadlock identified from #9785; I'm hoping to incorporate the synthetic deadlock test in a near future PR to prevent regressions ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - Tests were already in-place covering these fields; they've been adjusted from using the fields to the new accessor methods. - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [x] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title. <!--start release-notes-assistant--> ## Release notes <!--URL:https://codeberg.org/forgejo/forgejo--> - Bug fixes - [PR](https://codeberg.org/forgejo/forgejo/pulls/9922): <!--number 9922 --><!--line 0 --><!--description cmVkdWNlIGRlYWRsb2NrcyBtZXJnaW5nIFBScyBieSB1c2luZyBjYWNoaW5nIGZvciByZXBvIGlzc3VlIGNvdW50IHN0YXRz-->reduce deadlocks merging PRs by using caching for repo issue count stats<!--description--> <!--end release-notes-assistant--> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/9922 Reviewed-by: Gusted <gusted@noreply.codeberg.org> Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
2025-10-31 18:50:05 -04:00
if conn == nil || setting.CacheService.TTL <= 0 {
return getFunc()
}
cached := conn.Get(key)
if cached == nil {
value, err := getFunc()
2019-06-12 15:41:28 -04:00
if err != nil {
return value, err
2019-06-12 15:41:28 -04:00
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
switch v := conn.Get(key).(type) {
case int64:
return v, nil
case string:
value, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return 0, err
}
return value, nil
default:
value, err := getFunc()
if err != nil {
return value, err
}
return value, conn.Put(key, value, setting.CacheService.TTLSeconds())
}
}
// Remove key from cache
func Remove(key string) {
if conn == nil {
return
}
2019-06-12 15:41:28 -04:00
_ = conn.Delete(key)
}