Support localized README (#20508)

* Support localized README

* Slightly simplify getting the readme file and add some tests. Ensure that i18n also
works for docs/ etc.

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Update modules/markup/renderer.go

* Update modules/markup/renderer.go

* Update modules/markup/renderer.go

Co-authored-by: Andrew Thornton <art27@cantab.net>

modules/markup/renderer.go

@@ -310,14 +310,9 @@ func IsMarkupFile(name, markup string) bool {
 }
 
 // IsReadmeFile reports whether name looks like a README file
-// based on its name. If an extension is provided, it will strictly
+// based on its name.
-// match that extension.
+func IsReadmeFile(name string) bool {
-// Note that the '.' should be provided in ext, e.g ".md"
-func IsReadmeFile(name string, ext ...string) bool {
 	name = strings.ToLower(name)
-	if len(ext) > 0 {
-		return name == "readme"+ext[0]
-	}
 	if len(name) < 6 {
 		return false
 	} else if len(name) == 6 {
@@ -325,3 +320,27 @@ func IsReadmeFile(name string, ext ...string) bool {
 	}
 	return name[:7] == "readme."
 }
+
+// IsReadmeFileExtension reports whether name looks like a README file
+// based on its name. It will look through the provided extensions and check if the file matches
+// one of the extensions and provide the index in the extension list.
+// If the filename is `readme.` with an unmatched extension it will match with the index equaling
+// the length of the provided extension list.
+// Note that the '.' should be provided in ext, e.g ".md"
+func IsReadmeFileExtension(name string, ext ...string) (int, bool) {
+	if len(name) < 6 || name[:6] != "readme" {
+		return 0, false
+	}
+
+	for i, extension := range ext {
+		if name[6:] == extension {
+			return i, true
+		}
+	}
+
+	if name[6] == '.' {
+		return len(ext), true
+	}
+
+	return 0, false
+}

modules/markup/renderer_test.go

@@ -40,24 +40,47 @@ func TestMisc_IsReadmeFile(t *testing.T) {
 		assert.False(t, IsReadmeFile(testCase))
 	}
 
-	trueTestCasesStrict := [][]string{
+	type extensionTestcase struct {
-		{"readme", ""},
+		name     string
-		{"readme.md", ".md"},
+		expected bool
-		{"readme.txt", ".txt"},
+		idx      int
-	}
-	falseTestCasesStrict := [][]string{
-		{"readme", ".md"},
-		{"readme.md", ""},
-		{"readme.md", ".txt"},
-		{"readme.md", "md"},
-		{"readmee.md", ".md"},
-		{"readme.i18n.md", ".md"},
 	}
 
-	for _, testCase := range trueTestCasesStrict {
+	exts := []string{".md", ".txt", ""}
-		assert.True(t, IsReadmeFile(testCase[0], testCase[1]))
+	testCasesExtensions := []extensionTestcase{
+		{
+			name:     "readme",
+			expected: true,
+			idx:      2,
+		},
+		{
+			name:     "readme.md",
+			expected: true,
+			idx:      0,
+		},
+		{
+			name:     "readme.txt",
+			expected: true,
+			idx:      1,
+		},
+		{
+			name:     "readme.doc",
+			expected: true,
+			idx:      3,
+		},
+		{
+			name: "readmee.md",
+		},
+		{
+			name:     "readme..",
+			expected: true,
+			idx:      3,
+		},
 	}
-	for _, testCase := range falseTestCasesStrict {
+
-		assert.False(t, IsReadmeFile(testCase[0], testCase[1]))
+	for _, testCase := range testCasesExtensions {
+		idx, ok := IsReadmeFileExtension(testCase.name, exts...)
+		assert.Equal(t, testCase.expected, ok)
+		assert.Equal(t, testCase.idx, idx)
 	}
 }

routers/web/repo/view.go

@@ -57,7 +57,7 @@ type namedBlob struct {
 }
 
 // FIXME: There has to be a more efficient way of doing this
-func getReadmeFileFromPath(commit *git.Commit, treePath string) (*namedBlob, error) {
+func getReadmeFileFromPath(ctx *context.Context, commit *git.Commit, treePath string) (*namedBlob, error) {
 	tree, err := commit.SubTree(treePath)
 	if err != nil {
 		return nil, err
@@ -68,14 +68,18 @@ func getReadmeFileFromPath(commit *git.Commit, treePath string) (*namedBlob, err
 		return nil, err
 	}
 
-	var readmeFiles [4]*namedBlob
+	// Create a list of extensions in priority order
-	exts := []string{".md", ".txt", ""} // sorted by priority
+	// 1. Markdown files - with and without localisation - e.g. README.en-us.md or README.md
+	// 2. Txt files - e.g. README.txt
+	// 3. No extension - e.g. README
+	exts := append(localizedExtensions(".md", ctx.Language()), ".txt", "") // sorted by priority
+	extCount := len(exts)
+	readmeFiles := make([]*namedBlob, extCount+1)
 	for _, entry := range entries {
 		if entry.IsDir() {
 			continue
 		}
-		for i, ext := range exts {
+		if i, ok := markup.IsReadmeFileExtension(entry.Name(), exts...); ok {
-			if markup.IsReadmeFile(entry.Name(), ext) {
 			if readmeFiles[i] == nil || base.NaturalSortLess(readmeFiles[i].name, entry.Blob().Name()) {
 				name := entry.Name()
 				isSymlink := entry.IsLink()
@@ -96,27 +100,6 @@ func getReadmeFileFromPath(commit *git.Commit, treePath string) (*namedBlob, err
 			}
 		}
 	}
-
-		if markup.IsReadmeFile(entry.Name()) {
-			if readmeFiles[3] == nil || base.NaturalSortLess(readmeFiles[3].name, entry.Blob().Name()) {
-				name := entry.Name()
-				isSymlink := entry.IsLink()
-				if isSymlink {
-					entry, err = entry.FollowLinks()
-					if err != nil && !git.IsErrBadLink(err) {
-						return nil, err
-					}
-				}
-				if entry != nil && (entry.IsExecutable() || entry.IsRegular()) {
-					readmeFiles[3] = &namedBlob{
-						name,
-						isSymlink,
-						entry.Blob(),
-					}
-				}
-			}
-		}
-	}
 	var readmeFile *namedBlob
 	for _, f := range readmeFiles {
 		if f != nil {
@@ -151,13 +134,38 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 	renderReadmeFile(ctx, readmeFile, readmeTreelink)
 }
 
+// localizedExtensions prepends the provided language code with and without a
+// regional identifier to the provided extenstion.
+// Note: the language code will always be lower-cased, if a region is present it must be separated with a `-`
+// Note: ext should be prefixed with a `.`
+func localizedExtensions(ext, languageCode string) (localizedExts []string) {
+	if len(languageCode) < 1 {
+		return []string{ext}
+	}
+
+	lowerLangCode := "." + strings.ToLower(languageCode)
+
+	if strings.Contains(lowerLangCode, "-") {
+		underscoreLangCode := strings.ReplaceAll(lowerLangCode, "-", "_")
+		indexOfDash := strings.Index(lowerLangCode, "-")
+		// e.g. [.zh-cn.md, .zh_cn.md, .zh.md, .md]
+		return []string{lowerLangCode + ext, underscoreLangCode + ext, lowerLangCode[:indexOfDash] + ext, ext}
+	}
+
+	// e.g. [.en.md, .md]
+	return []string{lowerLangCode + ext, ext}
+}
+
 func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string) (*namedBlob, string) {
-	// 3 for the extensions in exts[] in order
+	// Create a list of extensions in priority order
-	// the last one is for a readme that doesn't
+	// 1. Markdown files - with and without localisation - e.g. README.en-us.md or README.md
-	// strictly match an extension
+	// 2. Txt files - e.g. README.txt
-	var readmeFiles [4]*namedBlob
+	// 3. No extension - e.g. README
-	var docsEntries [3]*git.TreeEntry
+	exts := append(localizedExtensions(".md", ctx.Language()), ".txt", "") // sorted by priority
-	exts := []string{".md", ".txt", ""} // sorted by priority
+	extCount := len(exts)
+	readmeFiles := make([]*namedBlob, extCount+1)
+
+	docsEntries := make([]*git.TreeEntry, 3) // (one of docs/, .gitea/ or .github/)
 	for _, entry := range entries {
 		if entry.IsDir() {
 			lowerName := strings.ToLower(entry.Name())
@@ -178,9 +186,8 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string)
 			continue
 		}
 
-		for i, ext := range exts {
+		if i, ok := markup.IsReadmeFileExtension(entry.Name(), exts...); ok {
-			if markup.IsReadmeFile(entry.Name(), ext) {
+			log.Debug("Potential readme file: %s", entry.Name())
-				log.Debug("%s", entry.Name())
 			name := entry.Name()
 			isSymlink := entry.IsLink()
 			target := entry
@@ -192,7 +199,6 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string)
 					return nil, ""
 				}
 			}
-				log.Debug("%t", target == nil)
 			if target != nil && (target.IsExecutable() || target.IsRegular()) {
 				readmeFiles[i] = &namedBlob{
 					name,
@@ -203,27 +209,6 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string)
 		}
 	}
 
-		if markup.IsReadmeFile(entry.Name()) {
-			name := entry.Name()
-			isSymlink := entry.IsLink()
-			if isSymlink {
-				var err error
-				entry, err = entry.FollowLinks()
-				if err != nil && !git.IsErrBadLink(err) {
-					ctx.ServerError("FollowLinks", err)
-					return nil, ""
-				}
-			}
-			if entry != nil && (entry.IsExecutable() || entry.IsRegular()) {
-				readmeFiles[3] = &namedBlob{
-					name,
-					isSymlink,
-					entry.Blob(),
-				}
-			}
-		}
-	}
-
 	var readmeFile *namedBlob
 	readmeTreelink := treeLink
 	for _, f := range readmeFiles {
@@ -239,7 +224,7 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string)
 				continue
 			}
 			var err error
-			readmeFile, err = getReadmeFileFromPath(ctx.Repo.Commit, entry.GetSubJumpablePathName())
+			readmeFile, err = getReadmeFileFromPath(ctx, ctx.Repo.Commit, entry.GetSubJumpablePathName())
 			if err != nil {
 				ctx.ServerError("getReadmeFileFromPath", err)
 				return nil, ""

routers/web/repo/view_test.go

@@ -0,0 +1,63 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repo
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_localizedExtensions(t *testing.T) {
+	tests := []struct {
+		name              string
+		ext               string
+		languageCode      string
+		wantLocalizedExts []string
+	}{
+		{
+			name:              "empty language",
+			ext:               ".md",
+			wantLocalizedExts: []string{".md"},
+		},
+		{
+			name:              "No region - lowercase",
+			languageCode:      "en",
+			ext:               ".csv",
+			wantLocalizedExts: []string{".en.csv", ".csv"},
+		},
+		{
+			name:              "No region - uppercase",
+			languageCode:      "FR",
+			ext:               ".txt",
+			wantLocalizedExts: []string{".fr.txt", ".txt"},
+		},
+		{
+			name:              "With region - lowercase",
+			languageCode:      "en-us",
+			ext:               ".md",
+			wantLocalizedExts: []string{".en-us.md", ".en_us.md", ".en.md", ".md"},
+		},
+		{
+			name:              "With region - uppercase",
+			languageCode:      "en-CA",
+			ext:               ".MD",
+			wantLocalizedExts: []string{".en-ca.MD", ".en_ca.MD", ".en.MD", ".MD"},
+		},
+		{
+			name:              "With region - all uppercase",
+			languageCode:      "ZH-TW",
+			ext:               ".md",
+			wantLocalizedExts: []string{".zh-tw.md", ".zh_tw.md", ".zh.md", ".md"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotLocalizedExts := localizedExtensions(tt.ext, tt.languageCode); !reflect.DeepEqual(gotLocalizedExts, tt.wantLocalizedExts) {
+				t.Errorf("localizedExtensions() = %v, want %v", gotLocalizedExts, tt.wantLocalizedExts)
+			}
+		})
+	}
+}