diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d88c7529fd..20ff31054d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,13 +8,17 @@ name: Tests # Polish.json still reached main, so the suite also needs to gate merges here. # # This covers the translation files as well - UITests contains LanguageJsonFilesTests and -# SeLanguageJsonContextTests - which is why there is no separate language-file workflow: -# building UITests is nearly the whole cost of this job, so a second workflow that builds -# the same project to run 66 of its tests would roughly double CI time for no extra -# coverage. +# SeLanguageJsonContextTests - which is why there is no separate language-file workflow. # -# `dotnet test` on the solution picks up the four test projects (libse, libuilogic, -# seconv, UI); the benchmark projects are not test projects and are skipped. +# The suite is split over parallel jobs because UITests is nearly all of the run time and +# cannot parallelize inside one process: every headless test shares one Avalonia dispatcher +# thread and the static Se.Settings, so tests/xunit.runner.json turns parallelization off. +# Separate processes have none of that shared state, so UITests is sharded by namespace +# into three runs of about the same length (measured 10-20: 118 s in one process, 48 s as +# three). The three filters partition the namespace - Features.Main, the rest of Features, +# everything else - so a new test lands in exactly one shard without editing this file. +# The libs job builds the whole solution, so the app and benchmark projects still have to +# compile; the UI jobs build only UITests and what it references. on: pull_request: push: @@ -24,8 +28,34 @@ on: jobs: test: + name: test (${{ matrix.name }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: libs + build: SubtitleEdit.sln + projects: tests/libse/LibSETests.csproj tests/libuilogic/LibUiLogicTests.csproj tests/seconv/SeConvTests.csproj + filter: '' + - name: ui-main + build: tests/UI/UITests.csproj + projects: tests/UI/UITests.csproj + filter: FullyQualifiedName~UITests.Features.Main. + - name: ui-features + build: tests/UI/UITests.csproj + projects: tests/UI/UITests.csproj + filter: FullyQualifiedName~UITests.Features.&FullyQualifiedName!~UITests.Features.Main. + - name: ui-other + build: tests/UI/UITests.csproj + projects: tests/UI/UITests.csproj + filter: FullyQualifiedName!~UITests.Features. + + env: + PROJECTS: ${{ matrix.projects }} + FILTER: ${{ matrix.filter }} + steps: - uses: actions/checkout@v5 @@ -35,15 +65,21 @@ jobs: dotnet-version: '10.0.x' - name: Restore - run: dotnet restore SubtitleEdit.sln + run: dotnet restore ${{ matrix.build }} - name: Build - run: dotnet build SubtitleEdit.sln --no-restore + run: dotnet build ${{ matrix.build }} --no-restore + # Every project runs even when an earlier one fails, so one run reports all failures. - name: Test id: test continue-on-error: true - run: dotnet test SubtitleEdit.sln --no-build --logger "trx;LogFileName=tests.trx" + run: | + status=0 + for p in $PROJECTS; do + dotnet test "$p" --no-build --logger "trx;LogFileName=tests.trx" ${FILTER:+--filter "$FILTER"} || status=1 + done + exit $status # The headless UI suite has a contamination failure mode no in-test fix can cover: # when a test crashes the shared Avalonia dispatcher thread, every later test that @@ -56,7 +92,11 @@ jobs: if: steps.test.outcome == 'failure' run: | echo "::warning title=Flaky test run::First test run failed; retrying once in a fresh process. Check the test-results artifact for the failure." - dotnet test SubtitleEdit.sln --no-build --logger "trx;LogFileName=tests-retry.trx" + status=0 + for p in $PROJECTS; do + dotnet test "$p" --no-build --logger "trx;LogFileName=tests-retry.trx" ${FILTER:+--filter "$FILTER"} || status=1 + done + exit $status # Uploaded whenever the first run failed - also on a green retry, so flakes can be # diagnosed without re-running locally. Needs !cancelled(): without a status function @@ -66,6 +106,6 @@ jobs: if: ${{ !cancelled() && steps.test.outcome == 'failure' }} uses: actions/upload-artifact@v5 with: - name: test-results + name: test-results-${{ matrix.name }} path: '**/TestResults/*.trx' if-no-files-found: ignore diff --git a/Dictionaries/dan_OCRFixReplaceList.xml b/Dictionaries/dan_OCRFixReplaceList.xml index 1b8b11be61e..71f99c08a6e 100644 --- a/Dictionaries/dan_OCRFixReplaceList.xml +++ b/Dictionaries/dan_OCRFixReplaceList.xml @@ -600,14 +600,14 @@ - - - - + + + + @@ -670,7 +670,7 @@ - + diff --git a/Dictionaries/deu_OCRFixReplaceList.xml b/Dictionaries/deu_OCRFixReplaceList.xml index 64d029a8524..a2b7f11eb9d 100644 --- a/Dictionaries/deu_OCRFixReplaceList.xml +++ b/Dictionaries/deu_OCRFixReplaceList.xml @@ -3933,7 +3933,6 @@ - @@ -6499,8 +6498,6 @@ - - @@ -6509,7 +6506,6 @@ - @@ -6982,12 +6978,8 @@ - - - - diff --git a/Dictionaries/eng_OCRFixReplaceList.xml b/Dictionaries/eng_OCRFixReplaceList.xml index 65f39943d7b..9222032e037 100644 --- a/Dictionaries/eng_OCRFixReplaceList.xml +++ b/Dictionaries/eng_OCRFixReplaceList.xml @@ -696,7 +696,6 @@ - @@ -1064,7 +1063,6 @@ - @@ -1289,7 +1287,6 @@ - @@ -1563,7 +1560,6 @@ - @@ -2342,7 +2338,6 @@ - diff --git a/Dictionaries/hrv_OCRFixReplaceList.xml b/Dictionaries/hrv_OCRFixReplaceList.xml index 318dcc672c8..3d4c5b8e2cb 100644 --- a/Dictionaries/hrv_OCRFixReplaceList.xml +++ b/Dictionaries/hrv_OCRFixReplaceList.xml @@ -7033,7 +7033,7 @@ - + @@ -7102,10 +7102,10 @@ - + - + diff --git a/Dictionaries/hun_OCRFixReplaceList.xml b/Dictionaries/hun_OCRFixReplaceList.xml index dc7c06bee46..68aac9143a5 100644 --- a/Dictionaries/hun_OCRFixReplaceList.xml +++ b/Dictionaries/hun_OCRFixReplaceList.xml @@ -21,17 +21,14 @@ - + - - - + + + - - - @@ -44,6 +41,8 @@ + + diff --git a/Dictionaries/ita_OCRFixReplaceList.xml b/Dictionaries/ita_OCRFixReplaceList.xml index 94411f1b6eb..cf37f16b011 100644 --- a/Dictionaries/ita_OCRFixReplaceList.xml +++ b/Dictionaries/ita_OCRFixReplaceList.xml @@ -8,9 +8,6 @@ - - - @@ -26,6 +23,9 @@ + + + @@ -61,7 +61,7 @@ - + diff --git a/Dictionaries/nld_OCRFixReplaceList.xml b/Dictionaries/nld_OCRFixReplaceList.xml index 502b7ed688c..f34f31bc080 100644 --- a/Dictionaries/nld_OCRFixReplaceList.xml +++ b/Dictionaries/nld_OCRFixReplaceList.xml @@ -10,17 +10,28 @@ - - - - + + + + + + + + + + + + + + + @@ -32,9 +43,6 @@ - - - @@ -60,7 +68,6 @@ - @@ -85,24 +92,15 @@ - - - - - - - - - @@ -124,7 +122,6 @@ - @@ -138,8 +135,6 @@ - - @@ -168,6 +163,7 @@ + @@ -343,6 +339,17 @@ + + + + + + + + + + + diff --git a/Dictionaries/nob_OCRFixReplaceList.xml b/Dictionaries/nob_OCRFixReplaceList.xml index 3cc8f9d29db..d93a09b9bf8 100644 --- a/Dictionaries/nob_OCRFixReplaceList.xml +++ b/Dictionaries/nob_OCRFixReplaceList.xml @@ -115,7 +115,7 @@ - + @@ -126,7 +126,7 @@ - + diff --git a/Dictionaries/nor_OCRFixReplaceList.xml b/Dictionaries/nor_OCRFixReplaceList.xml index 556c86f8f48..6bd0d800c60 100644 --- a/Dictionaries/nor_OCRFixReplaceList.xml +++ b/Dictionaries/nor_OCRFixReplaceList.xml @@ -113,7 +113,7 @@ - + @@ -124,7 +124,7 @@ - + diff --git a/Dictionaries/por_OCRFixReplaceList.xml b/Dictionaries/por_OCRFixReplaceList.xml index 32d517c32a9..4b6e1b8de0a 100644 --- a/Dictionaries/por_OCRFixReplaceList.xml +++ b/Dictionaries/por_OCRFixReplaceList.xml @@ -411,7 +411,6 @@ - @@ -562,7 +561,7 @@ - + diff --git a/Dictionaries/swe_OCRFixReplaceList.xml b/Dictionaries/swe_OCRFixReplaceList.xml index b11ed1041ab..bb362572fc6 100644 --- a/Dictionaries/swe_OCRFixReplaceList.xml +++ b/Dictionaries/swe_OCRFixReplaceList.xml @@ -290,7 +290,6 @@ - @@ -469,7 +468,7 @@ - + @@ -531,7 +530,7 @@ - + diff --git a/Themes/MidnightCinema/AssaAttachments.png b/Themes/MidnightCinema/AssaAttachments.png new file mode 100644 index 00000000000..fd13a364fdd Binary files /dev/null and b/Themes/MidnightCinema/AssaAttachments.png differ diff --git a/Themes/MidnightCinema/AssaDraw.png b/Themes/MidnightCinema/AssaDraw.png new file mode 100644 index 00000000000..0c0becfb181 Binary files /dev/null and b/Themes/MidnightCinema/AssaDraw.png differ diff --git a/Themes/MidnightCinema/AssaProperties.png b/Themes/MidnightCinema/AssaProperties.png new file mode 100644 index 00000000000..697028218cb Binary files /dev/null and b/Themes/MidnightCinema/AssaProperties.png differ diff --git a/Themes/MidnightCinema/AssaStyle.png b/Themes/MidnightCinema/AssaStyle.png new file mode 100644 index 00000000000..99e221384cd Binary files /dev/null and b/Themes/MidnightCinema/AssaStyle.png differ diff --git a/Themes/MidnightCinema/AutoTranslate.png b/Themes/MidnightCinema/AutoTranslate.png new file mode 100644 index 00000000000..545a826333b Binary files /dev/null and b/Themes/MidnightCinema/AutoTranslate.png differ diff --git a/Themes/MidnightCinema/BeautifyTimeCodes.png b/Themes/MidnightCinema/BeautifyTimeCodes.png new file mode 100644 index 00000000000..b1f1844fd64 Binary files /dev/null and b/Themes/MidnightCinema/BeautifyTimeCodes.png differ diff --git a/Themes/MidnightCinema/BurnIn.png b/Themes/MidnightCinema/BurnIn.png new file mode 100644 index 00000000000..9b7e913ad63 Binary files /dev/null and b/Themes/MidnightCinema/BurnIn.png differ diff --git a/Themes/MidnightCinema/EbuProperties.png b/Themes/MidnightCinema/EbuProperties.png new file mode 100644 index 00000000000..24bbc6e6b7d Binary files /dev/null and b/Themes/MidnightCinema/EbuProperties.png differ diff --git a/Themes/MidnightCinema/Error.png b/Themes/MidnightCinema/Error.png new file mode 100644 index 00000000000..aeae9e235dc Binary files /dev/null and b/Themes/MidnightCinema/Error.png differ diff --git a/Themes/MidnightCinema/Find.png b/Themes/MidnightCinema/Find.png new file mode 100644 index 00000000000..561f4e721d1 Binary files /dev/null and b/Themes/MidnightCinema/Find.png differ diff --git a/Themes/MidnightCinema/FixCommonErrors.png b/Themes/MidnightCinema/FixCommonErrors.png new file mode 100644 index 00000000000..35fee4bffb8 Binary files /dev/null and b/Themes/MidnightCinema/FixCommonErrors.png differ diff --git a/Themes/MidnightCinema/Help.png b/Themes/MidnightCinema/Help.png new file mode 100644 index 00000000000..b9d61b55101 Binary files /dev/null and b/Themes/MidnightCinema/Help.png differ diff --git a/Themes/MidnightCinema/Information.png b/Themes/MidnightCinema/Information.png new file mode 100644 index 00000000000..ccc1058469a Binary files /dev/null and b/Themes/MidnightCinema/Information.png differ diff --git a/Themes/MidnightCinema/IttProperties.png b/Themes/MidnightCinema/IttProperties.png new file mode 100644 index 00000000000..c0d283d86f5 Binary files /dev/null and b/Themes/MidnightCinema/IttProperties.png differ diff --git a/Themes/MidnightCinema/Layout.png b/Themes/MidnightCinema/Layout.png new file mode 100644 index 00000000000..8cf68cc8852 Binary files /dev/null and b/Themes/MidnightCinema/Layout.png differ diff --git a/Themes/MidnightCinema/MultipleReplace.png b/Themes/MidnightCinema/MultipleReplace.png new file mode 100644 index 00000000000..8919ac81e02 Binary files /dev/null and b/Themes/MidnightCinema/MultipleReplace.png differ diff --git a/Themes/MidnightCinema/Netflix.png b/Themes/MidnightCinema/Netflix.png new file mode 100644 index 00000000000..b29e24087af Binary files /dev/null and b/Themes/MidnightCinema/Netflix.png differ diff --git a/Themes/MidnightCinema/New.png b/Themes/MidnightCinema/New.png new file mode 100644 index 00000000000..7428e691eff Binary files /dev/null and b/Themes/MidnightCinema/New.png differ diff --git a/Themes/MidnightCinema/Open.png b/Themes/MidnightCinema/Open.png new file mode 100644 index 00000000000..10146641c42 Binary files /dev/null and b/Themes/MidnightCinema/Open.png differ diff --git a/Themes/MidnightCinema/OpenVideo.png b/Themes/MidnightCinema/OpenVideo.png new file mode 100644 index 00000000000..e824b405e7b Binary files /dev/null and b/Themes/MidnightCinema/OpenVideo.png differ diff --git a/Themes/MidnightCinema/PointSync.png b/Themes/MidnightCinema/PointSync.png new file mode 100644 index 00000000000..88a25ad0eb3 Binary files /dev/null and b/Themes/MidnightCinema/PointSync.png differ diff --git a/Themes/MidnightCinema/Question.png b/Themes/MidnightCinema/Question.png new file mode 100644 index 00000000000..b9d61b55101 Binary files /dev/null and b/Themes/MidnightCinema/Question.png differ diff --git a/Themes/MidnightCinema/RemoveTextForHi.png b/Themes/MidnightCinema/RemoveTextForHi.png new file mode 100644 index 00000000000..f4130940947 Binary files /dev/null and b/Themes/MidnightCinema/RemoveTextForHi.png differ diff --git a/Themes/MidnightCinema/Replace.png b/Themes/MidnightCinema/Replace.png new file mode 100644 index 00000000000..80ea36fed95 Binary files /dev/null and b/Themes/MidnightCinema/Replace.png differ diff --git a/Themes/MidnightCinema/Save.png b/Themes/MidnightCinema/Save.png new file mode 100644 index 00000000000..cf9948f4f8f Binary files /dev/null and b/Themes/MidnightCinema/Save.png differ diff --git a/Themes/MidnightCinema/SaveAs.png b/Themes/MidnightCinema/SaveAs.png new file mode 100644 index 00000000000..cf403a7bb4a Binary files /dev/null and b/Themes/MidnightCinema/SaveAs.png differ diff --git a/Themes/MidnightCinema/Settings.png b/Themes/MidnightCinema/Settings.png new file mode 100644 index 00000000000..c688e460fa5 Binary files /dev/null and b/Themes/MidnightCinema/Settings.png differ diff --git a/Themes/MidnightCinema/SourceView.png b/Themes/MidnightCinema/SourceView.png new file mode 100644 index 00000000000..bdc94db24d6 Binary files /dev/null and b/Themes/MidnightCinema/SourceView.png differ diff --git a/Themes/MidnightCinema/SpeechToText.png b/Themes/MidnightCinema/SpeechToText.png new file mode 100644 index 00000000000..71e78c42a39 Binary files /dev/null and b/Themes/MidnightCinema/SpeechToText.png differ diff --git a/Themes/MidnightCinema/SpellCheck.png b/Themes/MidnightCinema/SpellCheck.png new file mode 100644 index 00000000000..ed8b2a16ab0 Binary files /dev/null and b/Themes/MidnightCinema/SpellCheck.png differ diff --git a/Themes/MidnightCinema/VideoBackground.jpg b/Themes/MidnightCinema/VideoBackground.jpg new file mode 100644 index 00000000000..4a9f400fb37 Binary files /dev/null and b/Themes/MidnightCinema/VideoBackground.jpg differ diff --git a/Themes/MidnightCinema/VisualSync.png b/Themes/MidnightCinema/VisualSync.png new file mode 100644 index 00000000000..ed8ced095ff Binary files /dev/null and b/Themes/MidnightCinema/VisualSync.png differ diff --git a/Themes/MidnightCinema/Warning.png b/Themes/MidnightCinema/Warning.png new file mode 100644 index 00000000000..ba2b9d8ec63 Binary files /dev/null and b/Themes/MidnightCinema/Warning.png differ diff --git a/Themes/MidnightCinema/WaveformBackground.png b/Themes/MidnightCinema/WaveformBackground.png new file mode 100644 index 00000000000..82ae3848c0c Binary files /dev/null and b/Themes/MidnightCinema/WaveformBackground.png differ diff --git a/Themes/MidnightCinema/WaveformCenter.png b/Themes/MidnightCinema/WaveformCenter.png new file mode 100644 index 00000000000..9d0375524c3 Binary files /dev/null and b/Themes/MidnightCinema/WaveformCenter.png differ diff --git a/Themes/MidnightCinema/WaveformPause.png b/Themes/MidnightCinema/WaveformPause.png new file mode 100644 index 00000000000..5d7bd5bb482 Binary files /dev/null and b/Themes/MidnightCinema/WaveformPause.png differ diff --git a/Themes/MidnightCinema/WaveformPlay.png b/Themes/MidnightCinema/WaveformPlay.png new file mode 100644 index 00000000000..d6a2753ee27 Binary files /dev/null and b/Themes/MidnightCinema/WaveformPlay.png differ diff --git a/Themes/MidnightCinema/WaveformPlayPause.png b/Themes/MidnightCinema/WaveformPlayPause.png new file mode 100644 index 00000000000..482ba52359e Binary files /dev/null and b/Themes/MidnightCinema/WaveformPlayPause.png differ diff --git a/Themes/MidnightCinema/WaveformPlaybackSpeed.png b/Themes/MidnightCinema/WaveformPlaybackSpeed.png new file mode 100644 index 00000000000..c1da7b59b28 Binary files /dev/null and b/Themes/MidnightCinema/WaveformPlaybackSpeed.png differ diff --git a/Themes/MidnightCinema/WaveformZoomIn.png b/Themes/MidnightCinema/WaveformZoomIn.png new file mode 100644 index 00000000000..c7b1649c9ff Binary files /dev/null and b/Themes/MidnightCinema/WaveformZoomIn.png differ diff --git a/Themes/MidnightCinema/WaveformZoomOut.png b/Themes/MidnightCinema/WaveformZoomOut.png new file mode 100644 index 00000000000..d87806e13bb Binary files /dev/null and b/Themes/MidnightCinema/WaveformZoomOut.png differ diff --git a/Themes/MidnightCinema/WebVttProperties.png b/Themes/MidnightCinema/WebVttProperties.png new file mode 100644 index 00000000000..0e04659e750 Binary files /dev/null and b/Themes/MidnightCinema/WebVttProperties.png differ diff --git a/Themes/MidnightCinema/WebVttStyle.png b/Themes/MidnightCinema/WebVttStyle.png new file mode 100644 index 00000000000..9215d3bd08b Binary files /dev/null and b/Themes/MidnightCinema/WebVttStyle.png differ diff --git a/Themes/Nord/VideoBackground.jpg b/Themes/Nord/VideoBackground.jpg new file mode 100644 index 00000000000..1550cb13d3b Binary files /dev/null and b/Themes/Nord/VideoBackground.jpg differ diff --git a/Themes/Nord/WaveformBackground.png b/Themes/Nord/WaveformBackground.png new file mode 100644 index 00000000000..034d958c4c8 Binary files /dev/null and b/Themes/Nord/WaveformBackground.png differ diff --git a/Themes/Pastel/VideoBackground.jpg b/Themes/Pastel/VideoBackground.jpg new file mode 100644 index 00000000000..8d47ae26342 Binary files /dev/null and b/Themes/Pastel/VideoBackground.jpg differ diff --git a/Themes/Pastel/WaveformBackground.png b/Themes/Pastel/WaveformBackground.png new file mode 100644 index 00000000000..f16b4b4ddfa Binary files /dev/null and b/Themes/Pastel/WaveformBackground.png differ diff --git a/Themes/Sakura/AssaAttachments.png b/Themes/Sakura/AssaAttachments.png new file mode 100644 index 00000000000..fca387d9e8d Binary files /dev/null and b/Themes/Sakura/AssaAttachments.png differ diff --git a/Themes/Sakura/AssaDraw.png b/Themes/Sakura/AssaDraw.png new file mode 100644 index 00000000000..a4d7b69a80e Binary files /dev/null and b/Themes/Sakura/AssaDraw.png differ diff --git a/Themes/Sakura/AssaProperties.png b/Themes/Sakura/AssaProperties.png new file mode 100644 index 00000000000..d20ab960910 Binary files /dev/null and b/Themes/Sakura/AssaProperties.png differ diff --git a/Themes/Sakura/AssaStyle.png b/Themes/Sakura/AssaStyle.png new file mode 100644 index 00000000000..c5367cc64fd Binary files /dev/null and b/Themes/Sakura/AssaStyle.png differ diff --git a/Themes/Sakura/AutoTranslate.png b/Themes/Sakura/AutoTranslate.png new file mode 100644 index 00000000000..7b9f3b27382 Binary files /dev/null and b/Themes/Sakura/AutoTranslate.png differ diff --git a/Themes/Sakura/BeautifyTimeCodes.png b/Themes/Sakura/BeautifyTimeCodes.png new file mode 100644 index 00000000000..db084275a03 Binary files /dev/null and b/Themes/Sakura/BeautifyTimeCodes.png differ diff --git a/Themes/Sakura/BurnIn.png b/Themes/Sakura/BurnIn.png new file mode 100644 index 00000000000..20e148d52c7 Binary files /dev/null and b/Themes/Sakura/BurnIn.png differ diff --git a/Themes/Sakura/EbuProperties.png b/Themes/Sakura/EbuProperties.png new file mode 100644 index 00000000000..bc15be700d1 Binary files /dev/null and b/Themes/Sakura/EbuProperties.png differ diff --git a/Themes/Sakura/Error.png b/Themes/Sakura/Error.png new file mode 100644 index 00000000000..61076729064 Binary files /dev/null and b/Themes/Sakura/Error.png differ diff --git a/Themes/Sakura/Find.png b/Themes/Sakura/Find.png new file mode 100644 index 00000000000..0b747b2d434 Binary files /dev/null and b/Themes/Sakura/Find.png differ diff --git a/Themes/Sakura/FixCommonErrors.png b/Themes/Sakura/FixCommonErrors.png new file mode 100644 index 00000000000..7005945bb42 Binary files /dev/null and b/Themes/Sakura/FixCommonErrors.png differ diff --git a/Themes/Sakura/Help.png b/Themes/Sakura/Help.png new file mode 100644 index 00000000000..805632ca587 Binary files /dev/null and b/Themes/Sakura/Help.png differ diff --git a/Themes/Sakura/Information.png b/Themes/Sakura/Information.png new file mode 100644 index 00000000000..872ad4f7515 Binary files /dev/null and b/Themes/Sakura/Information.png differ diff --git a/Themes/Sakura/IttProperties.png b/Themes/Sakura/IttProperties.png new file mode 100644 index 00000000000..a42c10d4100 Binary files /dev/null and b/Themes/Sakura/IttProperties.png differ diff --git a/Themes/Sakura/Layout.png b/Themes/Sakura/Layout.png new file mode 100644 index 00000000000..91d9f5c9892 Binary files /dev/null and b/Themes/Sakura/Layout.png differ diff --git a/Themes/Sakura/MultipleReplace.png b/Themes/Sakura/MultipleReplace.png new file mode 100644 index 00000000000..4b518512553 Binary files /dev/null and b/Themes/Sakura/MultipleReplace.png differ diff --git a/Themes/Sakura/Netflix.png b/Themes/Sakura/Netflix.png new file mode 100644 index 00000000000..db9122358a6 Binary files /dev/null and b/Themes/Sakura/Netflix.png differ diff --git a/Themes/Sakura/New.png b/Themes/Sakura/New.png new file mode 100644 index 00000000000..6166f0d7a33 Binary files /dev/null and b/Themes/Sakura/New.png differ diff --git a/Themes/Sakura/Open.png b/Themes/Sakura/Open.png new file mode 100644 index 00000000000..c31c25e7c41 Binary files /dev/null and b/Themes/Sakura/Open.png differ diff --git a/Themes/Sakura/OpenVideo.png b/Themes/Sakura/OpenVideo.png new file mode 100644 index 00000000000..701f2a39126 Binary files /dev/null and b/Themes/Sakura/OpenVideo.png differ diff --git a/Themes/Sakura/PointSync.png b/Themes/Sakura/PointSync.png new file mode 100644 index 00000000000..96e4121a217 Binary files /dev/null and b/Themes/Sakura/PointSync.png differ diff --git a/Themes/Sakura/Question.png b/Themes/Sakura/Question.png new file mode 100644 index 00000000000..805632ca587 Binary files /dev/null and b/Themes/Sakura/Question.png differ diff --git a/Themes/Sakura/RemoveTextForHi.png b/Themes/Sakura/RemoveTextForHi.png new file mode 100644 index 00000000000..1010a5aab5e Binary files /dev/null and b/Themes/Sakura/RemoveTextForHi.png differ diff --git a/Themes/Sakura/Replace.png b/Themes/Sakura/Replace.png new file mode 100644 index 00000000000..7b089bfc6e1 Binary files /dev/null and b/Themes/Sakura/Replace.png differ diff --git a/Themes/Sakura/Save.png b/Themes/Sakura/Save.png new file mode 100644 index 00000000000..48b97953351 Binary files /dev/null and b/Themes/Sakura/Save.png differ diff --git a/Themes/Sakura/SaveAs.png b/Themes/Sakura/SaveAs.png new file mode 100644 index 00000000000..fa7bf303a8a Binary files /dev/null and b/Themes/Sakura/SaveAs.png differ diff --git a/Themes/Sakura/Settings.png b/Themes/Sakura/Settings.png new file mode 100644 index 00000000000..f906e5644d3 Binary files /dev/null and b/Themes/Sakura/Settings.png differ diff --git a/Themes/Sakura/SourceView.png b/Themes/Sakura/SourceView.png new file mode 100644 index 00000000000..1e3ded621ce Binary files /dev/null and b/Themes/Sakura/SourceView.png differ diff --git a/Themes/Sakura/SpeechToText.png b/Themes/Sakura/SpeechToText.png new file mode 100644 index 00000000000..586b6e51865 Binary files /dev/null and b/Themes/Sakura/SpeechToText.png differ diff --git a/Themes/Sakura/SpellCheck.png b/Themes/Sakura/SpellCheck.png new file mode 100644 index 00000000000..7fec0bb0acd Binary files /dev/null and b/Themes/Sakura/SpellCheck.png differ diff --git a/Themes/Sakura/VideoBackground.jpg b/Themes/Sakura/VideoBackground.jpg new file mode 100644 index 00000000000..8e2c9ee5bc0 Binary files /dev/null and b/Themes/Sakura/VideoBackground.jpg differ diff --git a/Themes/Sakura/VisualSync.png b/Themes/Sakura/VisualSync.png new file mode 100644 index 00000000000..0f800da7b48 Binary files /dev/null and b/Themes/Sakura/VisualSync.png differ diff --git a/Themes/Sakura/Warning.png b/Themes/Sakura/Warning.png new file mode 100644 index 00000000000..d4df392cf15 Binary files /dev/null and b/Themes/Sakura/Warning.png differ diff --git a/Themes/Sakura/WaveformBackground.png b/Themes/Sakura/WaveformBackground.png new file mode 100644 index 00000000000..a9f54c7fb37 Binary files /dev/null and b/Themes/Sakura/WaveformBackground.png differ diff --git a/Themes/Sakura/WaveformCenter.png b/Themes/Sakura/WaveformCenter.png new file mode 100644 index 00000000000..8a635a3d5e5 Binary files /dev/null and b/Themes/Sakura/WaveformCenter.png differ diff --git a/Themes/Sakura/WaveformPause.png b/Themes/Sakura/WaveformPause.png new file mode 100644 index 00000000000..c862118ef8d Binary files /dev/null and b/Themes/Sakura/WaveformPause.png differ diff --git a/Themes/Sakura/WaveformPlay.png b/Themes/Sakura/WaveformPlay.png new file mode 100644 index 00000000000..0273ae84f91 Binary files /dev/null and b/Themes/Sakura/WaveformPlay.png differ diff --git a/Themes/Sakura/WaveformPlayPause.png b/Themes/Sakura/WaveformPlayPause.png new file mode 100644 index 00000000000..b20d073b1b5 Binary files /dev/null and b/Themes/Sakura/WaveformPlayPause.png differ diff --git a/Themes/Sakura/WaveformPlaybackSpeed.png b/Themes/Sakura/WaveformPlaybackSpeed.png new file mode 100644 index 00000000000..14d005c0b93 Binary files /dev/null and b/Themes/Sakura/WaveformPlaybackSpeed.png differ diff --git a/Themes/Sakura/WaveformZoomIn.png b/Themes/Sakura/WaveformZoomIn.png new file mode 100644 index 00000000000..5fbd9d25e48 Binary files /dev/null and b/Themes/Sakura/WaveformZoomIn.png differ diff --git a/Themes/Sakura/WaveformZoomOut.png b/Themes/Sakura/WaveformZoomOut.png new file mode 100644 index 00000000000..d7800ee548b Binary files /dev/null and b/Themes/Sakura/WaveformZoomOut.png differ diff --git a/Themes/Sakura/WebVttProperties.png b/Themes/Sakura/WebVttProperties.png new file mode 100644 index 00000000000..7e5eb206f99 Binary files /dev/null and b/Themes/Sakura/WebVttProperties.png differ diff --git a/Themes/Sakura/WebVttStyle.png b/Themes/Sakura/WebVttStyle.png new file mode 100644 index 00000000000..3c4f648821b Binary files /dev/null and b/Themes/Sakura/WebVttStyle.png differ diff --git a/change-log.txt b/change-log.txt index 46367a8923e..d1a530af4ff 100644 --- a/change-log.txt +++ b/change-log.txt @@ -3,6 +3,51 @@ Subtitle Edit Changelog ----------------------------------------------------------------------------------------------------- +v5.4.0-beta4 (21st of October 2026) + +* Add Midnight Cinema and Sakura themes, theme background art for video/waveform, Cycle theme shortcut +* Add Syntax coloring: color the gap when it falls inside a user-defined range (ms or frames) +* Add Set position: SE4-style click-to-place mode, toggle to hide selection box, remember window size - thx 114562175 +* Add Video player: optional playback speed button - thx benshgit +* Add Spell check: remove words from the dictionary (Word lists "Removed from dictionary") - thx fraternl +* Add setting: split at video and text box position keeps text at caret end - thx kadrimarzouki +* Copy from the grid includes the video offset, pasting SE's own copy removes it again +* One 13 px text size for all windows, main window in the editor look - thx GrampaWildWilly +* SubRip reader rewritten: tolerant of damaged files, 2.6-12x faster +* Set position: opens at once, frame rendered in the background, arrow keys nudge right away +* Batch convert: change frame rate saves at the target frame rate - thx michi987654 +* ITT: source header timing parameters follow the current frame rate - thx michi987654 +* OCR replace lists: fix rules that damage correct text (10 languages), Dutch GLM-OCR word errors - thx fraternl +* OCR fix: keep Dutch "Eén" - thx fraternl +* Text to speech: context line on its own row, long video names trimmed +* Performance: name casing, speech to text names, teletext, OCR splitter +* Fix crashes and hangs found by fuzzing subtitle format parsers +* Update Hungarian translation +* Update Italian translation - thx bovirus +* Update Japanese translation - thx hisui3393 +* Update Polish translation - thx potplayer-fanpack + +----------------------------------------------------------------------------------------------------- +v5.4.0-beta3 (20th of October 2026) + +* Add Settings: visual theme picker with preview cards +* Add Nord theme +* Add iTunes Timed Text: export presets, template-file header source and default color in File > Properties - thx michi987654 +* Add Speech to text: Danish fine-tuned Qwen3 ASR 0.6B model +* Add ASSA/SSA styles: category picker back, font name + color sample columns, resizable lists - thx bichitoxxx +* Multiple replace: category list + flat rules list with a type-in row, like SE4 - thx kadrimarzouki +* Editor look now follows the Font scale (%) setting - thx bichitoxxx +* Language auto-detect: everyday words for 30 languages, letter rules for Cyrillic (incl. Macedonian/Serbian), Baltic, Afrikaans/Dutch and Arabic script +* Qwen3 ASR: update qwen3-asr.cpp to v0.1.10 (mel filterbank fix, beam search), pass the selected language +* OCR fix engine: image-matcher I/l and spacing repairs, lone slash kept - thx fraternl +* More windows use the shared editor look helpers (button bars, check-box cells, video dialog switches) - thx ivandrofly +* Fix Qwen3 ASR "" marker in output with older engine builds +* Fix compact numeric field borders in the editor look - thx pdjdev +* Add Romanian translation - thx cosstell62-hash +* Update Italian translation - thx bovirus +* Update Japanese translation - thx hisui3393 + +----------------------------------------------------------------------------------------------------- v5.4.0-beta2 (19th of October 2026) * Add Multiple replace: "Add rules" window for typing a list of find/replace pairs, toolbar "New rule" button, F2 and Cmd shortcuts - thx kadrimarzouki diff --git a/docs/features/word-lists.md b/docs/features/word-lists.md index 523ef78149a..4bf0e14c7a2 100644 --- a/docs/features/word-lists.md +++ b/docs/features/word-lists.md @@ -12,7 +12,7 @@ Manage custom word lists used by spell check and OCR error fixing. 1. Open **Options → Word lists...** 2. Select a language from the dropdown -3. Manage entries in the three word list categories +3. Manage entries in the four word list categories 4. Changes are saved automatically when entries are added or removed ## Word List Categories @@ -31,6 +31,15 @@ Custom words added to the user dictionary. These supplement the main spell check - Add a new word in the text field and click **Add** - Select a word and click **Remove** to delete it +### Removed From Dictionary + +Words the spell checker should treat as misspelled even though the dictionary knows them. Use this for rare words that usually hide an error - for example Dutch "weetje" or "watje", which in subtitles are mostly "weet je" / "wat je" with the space lost. + +- A removed word is flagged by spell check and is not offered as a suggestion +- OCR error fixing treats it as unknown, so rules like "weetje" → "weet je" can fix it +- Adding the word as a user word takes it off this list again (and the other way round) +- Stored in the `` section of the language's `_user.xml` file + ### OCR Fix Replacements Find-and-replace pairs used during OCR to fix common recognition errors (e.g., "rn" → "m", "0" → "O"). diff --git a/src/libse/Cea608/GetCcDataHelper.cs b/src/libse/Cea608/GetCcDataHelper.cs index 4367636911f..c18e5952c48 100644 --- a/src/libse/Cea608/GetCcDataHelper.cs +++ b/src/libse/Cea608/GetCcDataHelper.cs @@ -66,7 +66,15 @@ public static List GetCcData(Stream fs, ulong startPos, ulong size, bool return fieldData; } - var length = (int)size; + // The size comes from the file's sample table: a corrupt one asked the pool for more + // than the largest possible array (OutOfMemoryException). Nothing past the end of the + // stream can be read anyway. + if (fs.Length <= (long)startPos) + { + return fieldData; + } + + var length = (int)Math.Min(size, (ulong)(fs.Length - (long)startPos)); // Read the sample once instead of seeking and reading 5 bytes per NAL unit. // The walk below only ever moves forward inside the sample, so a single diff --git a/src/libse/Common/ActorConverter.cs b/src/libse/Common/ActorConverter.cs index 869ed29582d..e8a672dce21 100644 --- a/src/libse/Common/ActorConverter.cs +++ b/src/libse/Common/ActorConverter.cs @@ -15,7 +15,6 @@ public class ActorConverter public const int ProperCase = 3; private SubtitleFormat _subtitleFormat; - private string _languageCode; private NameList _namesList; private HashSet _nameListInclMulti; @@ -29,7 +28,6 @@ public class ActorConverter public ActorConverter(SubtitleFormat subtitleFormat, string languageCode) { _subtitleFormat = subtitleFormat; - _languageCode = languageCode; _namesList = new NameList(Configuration.DictionariesDirectory, languageCode, false, string.Empty); _nameListInclMulti = new HashSet(_namesList.GetAllNames(), StringComparer.OrdinalIgnoreCase); } diff --git a/src/libse/Common/DialogSplitMerge.cs b/src/libse/Common/DialogSplitMerge.cs index 4ba560e2af2..d34640d3ca5 100644 --- a/src/libse/Common/DialogSplitMerge.cs +++ b/src/libse/Common/DialogSplitMerge.cs @@ -13,6 +13,13 @@ public class DialogSplitMerge public bool SkipLineEndingCheck { get; set; } public static char GetDashChar() => '-'; + + /// + /// Every IsDialog variant needs a line starting with a dash, so text without one is + /// returned as is - without splitting it and stripping tags for the dialog checks. + /// + private static bool CannotBeDialog(string input) => + input != null && input.IndexOf(GetDashChar()) < 0 && input.IndexOf(GetAlternateDashChar()) < 0; public static char GetAlternateDashChar() => '‐'; // Unicode En Dash (\u2010) public string FixDashesAndSpaces(string input) @@ -27,6 +34,11 @@ public string FixDashesAndSpaces(string input, Paragraph p, Paragraph prev) public string FixSpaces(string input) { + if (CannotBeDialog(input)) + { + return input; + } + var lines = input.SplitToLines(); if (!IsDialog(lines)) { @@ -94,6 +106,11 @@ public string FixSpaces(string input) public string AddSpaces(string input) { + if (CannotBeDialog(input)) + { + return input; + } + var lines = input.SplitToLines(); if (!IsDialog(lines)) { @@ -140,6 +157,11 @@ public string AddSpaces(string input) public string RemoveSpaces(string input) { + if (CannotBeDialog(input)) + { + return input; + } + var lines = input.SplitToLines(); if (!IsDialog(lines)) { @@ -189,6 +211,11 @@ public string RemoveSpaces(string input) public string FixDashes(string input, Paragraph p, Paragraph prev) { + if (CannotBeDialog(input)) + { + return input; + } + var lines = input.SplitToLines(); if (!IsDialog(lines, p, prev)) { @@ -285,6 +312,11 @@ public string FixDashes(string input, Paragraph p, Paragraph prev) public string FixDashes(string input) { + if (CannotBeDialog(input)) + { + return input; + } + var lines = input.SplitToLines(); if (!IsDialog(lines)) { diff --git a/src/libse/Common/FixCasing.cs b/src/libse/Common/FixCasing.cs index bc4b0531322..960b5538657 100644 --- a/src/libse/Common/FixCasing.cs +++ b/src/libse/Common/FixCasing.cs @@ -20,6 +20,7 @@ public class FixCasing private readonly string _language; private readonly List _names; + private NameListIndex _nameIndex; public FixCasing(string language) { @@ -36,16 +37,17 @@ public FixCasing(string language) public void Fix(Subtitle subtitle) { var subCulture = GetCultureInfoFromLanguage(_language); + var names = _nameIndex ?? (_nameIndex = new NameListIndex(_names)); Paragraph last = null; foreach (var p in subtitle.Paragraphs) { if (last != null) { - p.Text = Fix(p.Text, last.Text, _names, subCulture, p.StartTime.TotalMilliseconds - last.EndTime.TotalMilliseconds); + p.Text = Fix(p.Text, last.Text, names, subCulture, p.StartTime.TotalMilliseconds - last.EndTime.TotalMilliseconds); } else { - p.Text = Fix(p.Text, string.Empty, _names, subCulture, 10000); + p.Text = Fix(p.Text, string.Empty, names, subCulture, 10000); } // fix casing of English alone i to I @@ -267,7 +269,7 @@ public static string FixStutter(string text) return sb.ToString(); } - private string Fix(string original, string lastLine, List nameList, CultureInfo subtitleCulture, double millisecondsFromLast) + private string Fix(string original, string lastLine, NameListIndex nameList, CultureInfo subtitleCulture, double millisecondsFromLast) { if (lastLine == null) { diff --git a/src/libse/Common/LanguageAutoDetect.cs b/src/libse/Common/LanguageAutoDetect.cs index 47c627d61ea..2192c5e8050 100644 --- a/src/libse/Common/LanguageAutoDetect.cs +++ b/src/libse/Common/LanguageAutoDetect.cs @@ -118,6 +118,272 @@ private static int GetCount(string text, string[] words) return CountWords(text).Get(words); } + /// + /// Tells the Cyrillic languages apart by spelling, which the shared short words + /// (не, да, за, нас, може, така, беше, ...) cannot: ј/љ/њ/џ are written only in + /// Macedonian and Serbian (ќ/ѓ/ѕ only in Macedonian, ћ/ђ only in Serbian), і/ї/є/ґ only + /// in Ukrainian, ы/э/ё only in Russian, and ъ is everywhere in Bulgarian but rare in + /// Russian and absent from the rest. + /// + /// The text to look at. + /// Word counts of the same text, for a Macedonian/Serbian tie; null to count on demand. + /// "mk", "sr", "uk", "bg", "ru", or null when the letters do not say. + private static string GetCyrillicLanguageByLetters(string text, WordListCounts? counts) + { + var macedonianOrSerbian = 0; // ј љ њ џ ќ ѓ ѕ ћ ђ + var macedonianOnly = 0; // ќ ѓ ѕ + var serbianOnly = 0; // ћ ђ + var ukrainianOnly = 0; // і ї є ґ + var russianOnly = 0; // ы э ё + var hardSign = 0; // ъ + var eastSlavic = 0; // ъ ы э ё й щ ь ю я - none of them in Macedonian or Serbian + foreach (var c in text) + { + switch (c) + { + case 'ј': case 'Ј': case 'љ': case 'Љ': case 'њ': case 'Њ': case 'џ': case 'Џ': + macedonianOrSerbian++; + break; + case 'ќ': case 'Ќ': case 'ѓ': case 'Ѓ': case 'ѕ': case 'Ѕ': + macedonianOrSerbian++; + macedonianOnly++; + break; + case 'ћ': case 'Ћ': case 'ђ': case 'Ђ': + macedonianOrSerbian++; + serbianOnly++; + break; + case 'і': case 'І': case 'ї': case 'Ї': case 'є': case 'Є': case 'ґ': case 'Ґ': + ukrainianOnly++; + break; + case 'ы': case 'Ы': case 'э': case 'Э': case 'ё': case 'Ё': + russianOnly++; + eastSlavic++; + break; + case 'ъ': case 'Ъ': + hardSign++; + eastSlavic++; + break; + case 'й': case 'Й': case 'щ': case 'Щ': case 'ь': case 'Ь': case 'ю': case 'Ю': case 'я': case 'Я': + eastSlavic++; + break; + } + } + + // Strictly more, so that a few misread letters in an OCR'ed file cannot flip a + // Bulgarian or Russian text, where ъ/й/я/ю are everywhere. + if (macedonianOrSerbian > 0 && macedonianOrSerbian > eastSlavic) + { + if (macedonianOnly != serbianOnly) + { + return macedonianOnly > serbianOnly ? "mk" : "sr"; + } + + // Neither ќ/ѓ/ѕ nor ћ/ђ, or as many of each (an OCR tie): the word lists decide. + var c2 = counts ?? CountWords(text); + var macedonianCount = c2.Get(AutoDetectWordsMacedonian); + var serbianCount = c2.Get(AutoDetectWordsSerbianCyrillic) + c2.Get(AutoDetectWordsSerbianCyrillicOnly); + return macedonianCount > serbianCount ? "mk" : "sr"; + } + + if (ukrainianOnly > 0 && ukrainianOnly > russianOnly + hardSign) + { + return "uk"; + } + + if (russianOnly != hardSign) + { + return russianOnly > hardSign ? "ru" : "bg"; + } + + return null; + } + + /// + /// Lithuanian or Latvian by their own letters: ą/ę/ė/į/ų are Lithuanian, ā/ē/ī/ģ/ķ/ļ/ņ/ō + /// Latvian (č/š/ž/ū are both). The word lists overlap (kas, kur, kad, ir, jau, ...). + /// + /// The text to look at. + /// "lt", "lv", or null when neither letter set is in the text. + private static string GetBalticLanguageByLetters(string text) + { + var lithuanian = 0; + var latvian = 0; + foreach (var c in text) + { + switch (c) + { + case 'ą': case 'Ą': case 'ę': case 'Ę': case 'ė': case 'Ė': case 'į': case 'Į': case 'ų': case 'Ų': + lithuanian++; + break; + case 'ā': case 'Ā': case 'ē': case 'Ē': case 'ī': case 'Ī': case 'ģ': case 'Ģ': case 'ķ': case 'Ķ': + case 'ļ': case 'Ļ': case 'ņ': case 'Ņ': case 'ō': case 'Ō': + latvian++; + break; + } + } + + if (lithuanian == latvian) + { + return null; + } + + return lithuanian > latvian ? "lt" : "lv"; + } + + /// + /// Afrikaans or Dutch by spelling: Dutch writes z and ij all the time (zijn, zo, mij, + /// hij, zij), Afrikaans has no z and no ij but y instead (sy, hy, my, jy, altyd). + /// + /// The text to look at. + /// "af", "nl", or null when the text has neither. + private static string GetAfrikaansOrDutchByLetters(string text) + { + var dutch = 0; + var afrikaans = 0; + var previous = '\0'; + foreach (var c in text) + { + if (c == 'z' || c == 'Z') + { + dutch++; + } + else if ((c == 'j' || c == 'J') && (previous == 'i' || previous == 'I')) + { + dutch++; + } + else if (c == 'y' || c == 'Y') + { + afrikaans++; + } + + previous = c; + } + + if (dutch == afrikaans) + { + return null; + } + + return afrikaans > dutch ? "af" : "nl"; + } + + /// + /// Arabic, Persian or Urdu by the letters each script adds: Persian and Urdu write + /// پ/چ/ژ/گ and the Persian ye/kaf (ی U+06CC, ک U+06A9), Arabic its own ye/kaf + /// (ي U+064A, ك U+0643) plus ة/ذ/ض/ظ/ء; Urdu alone has ٹ/ڈ/ڑ/ں/ے/ھ/ہ. + /// + /// The text to look at. + /// "ur", "fa", "ar", or null when the letters do not say. + private static string GetArabicScriptLanguageByLetters(string text) + { + var arabic = 0; + var persian = 0; + var urdu = 0; + foreach (var c in text) + { + switch (c) + { + case 'ي': case 'ك': case 'ة': case 'ذ': case 'ض': case 'ظ': case 'ء': + arabic++; + break; + case 'ی': case 'ک': case 'پ': case 'چ': case 'ژ': case 'گ': + persian++; + break; + case 'ٹ': case 'ڈ': case 'ڑ': case 'ں': case 'ے': case 'ھ': case 'ہ': case 'ۓ': + urdu++; + break; + } + } + + if (urdu > 0 && urdu * 4 >= persian) + { + return "ur"; + } + + if (persian == arabic) + { + return null; + } + + return persian > arabic ? "fa" : "ar"; + } + + /// + /// Latin-script languages with letters of their own, for the letter fallback when no + /// word list matched (a line or two). Shared letters are left out: ç (fr/pt/tr/ca/sq), + /// ş (ro/tr), ä/ö/ü (de/sv/fi/et), å (sv/da/no), æ/ø (da/no), ć/đ (hr/sr/bs), + /// č/š/ž (cs/sk/sl/hr/sr/lt/lv), õ (pt/et), ô (fr/sk), ą/ę (pl/lt). + /// + private static readonly KeyValuePair[] LatinUniqueLetters = + { + new KeyValuePair("pl", "łŁżŻźŹśŚńŃ"), + new KeyValuePair("cs", "řŘěĚůŮ"), + new KeyValuePair("sk", "ľĽŕŔ"), + new KeyValuePair("hu", "őŐűŰ"), + new KeyValuePair("ro", "ăĂșȘțȚţŢ"), + new KeyValuePair("tr", "ığĞİ"), + new KeyValuePair("lt", "ėĖįĮųŲ"), + new KeyValuePair("lv", "āĀēĒīĪģĢķĶļĻņŅ"), + new KeyValuePair("is", "þÞðÐ"), + new KeyValuePair("vi", "ơƠưƯạẠảẢấẤầẦẩẨẫẪậẬắẮằẰẳẲẵẴặẶẹẸẻẺẽẼếẾềỀểỂễỄệỆỉỈịỊọỌỏỎốỐồỒổỔỗỖộỘớỚờỜởỞỡỠợỢụỤủỦứỨừỪửỬữỮựỰỳỲỵỴỷỶỹỸ"), + new KeyValuePair("pt", "ãÃ"), + new KeyValuePair("es", "ñÑ¿¡"), + }; + + /// + /// The Latin-script language whose own letters occur most in the text; Albanian when + /// ë (which French has only in a few words) outnumbers the French accents. + /// + /// The text to look at. + /// A language code, or null when no language-specific letter is in the text. + private static string GetLatinLanguageByUniqueLetters(string text) + { + string best = null; + var bestCount = 0; + foreach (var pair in LatinUniqueLetters) + { + var count = 0; + foreach (var c in text) + { + if (pair.Value.IndexOf(c) >= 0) + { + count++; + } + } + + if (count > bestCount) + { + best = pair.Key; + bestCount = count; + } + } + + if (best == null) + { + var albanian = 0; + var french = 0; + foreach (var c in text) + { + switch (c) + { + case 'ë': case 'Ë': + albanian++; + break; + case 'é': case 'É': case 'è': case 'È': case 'ê': case 'Ê': case 'à': case 'À': case 'ù': case 'Ù': + french++; + break; + } + } + + if (albanian >= 2 && albanian > french) + { + return "sq"; + } + } + + return best; + } + private static int GetCountContains(string text, params string[] words) { var count = 0; @@ -182,8 +448,18 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) // Short disambiguation lists: words that belong to one language but not its close // neighbour, probed only when the neighbour's main list already scored. - private static readonly string[] AutoDetectWordsNorwegianOnly = { "ut", "deg", "meg", "merkelig", "mye", "spørre" }; - private static readonly string[] AutoDetectWordsDanishOnly = { "siger", "dig", "mig", "mærkelig", "tilbage", "spørge" }; + private static readonly string[] AutoDetectWordsNorwegianOnly = { "ut", "deg", "meg", "merkelig", "mye", "spørre", + "vært", "sa", "noe", "noen", "hva", "etter", "uten", "mellom", "ble", "blitt", "vet", "dere", "seg", "oss", "henne", "nei", "takk", "unnskyld", + "liker", "trenger", "sier", "fjor", "sjanse", "aldri", "siste", "neste", "annen", "sånn", "enn", "å", "igjen", "gjøre", "gjør", "kanskje", "bra", + "veldig", "gutt", "jente", "elva", "vann", "jobbet", "jobb", "lei", "fikk", "sett", "ditt", "mitt", "sitt", "vårt", "kveld", + "natt", "uke", "penger", "redd", "kjære", "kjenner", "tilbake", "gjennom", "nesten", "nettopp", "dessverre", "viktig", "riktig", "ferdig", "trøtt", "liten", "litt", "hjelp", "hjelper", "begynner", "finner", "tar", "gir", "spør", "forteller", "arbeidet", }; + private static readonly string[] AutoDetectWordsDanishOnly = { "siger", "dig", "mig", "mærkelig", "tilbage", "spørge", + "hvad", "noget", "nogen", "nu", "meget", "efter", "uden", "mellem", "gøre", "gør", "blev", "blevet", "sig", "os", "hende", "nej", "undskyld", "lide", + "behøver", "sidste", "næste", "anden", "sådan", "jer", "jeres", "dit", "mit", "sit", "vores", "ind", "ud", "sagde", "sige", "kender", + "kendt", "kære", "arbejde", "arbejder", "arbejdede", "åbner", "begynder", "finder", "tager", "giver", "lægger", "sætter", "sidder", "løber", "kører", + "hjælper", "hjælp", "bruger", "spørger", "fortæller", "svær", "lille", "pæn", "syg", "træt", "færdig", "rigtig", "fejl", "vigtig", "kedelig", + "pludselig", "desværre", "netop", "næsten", "gennem", "inden", "penge", "aften", "nat", "uge", "dreng", "pige", "vand", "ventede", "fik", "ked", + "mere", "bange", "vred", "håber", "væk", }; private static readonly string[] AutoDetectWordsFrenchOnly = { "[Cc]'est", "pas", "vous", "pour", "suis", "Pourquoi", "maison", "souviens", "quelque" }; private static readonly string[] AutoDetectWordsPortugueseOnly = { @@ -208,25 +484,49 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "strong", "simply", "situation","soldier", "somebody", "someone","something", "sometimes", "speak","statement", "perhaps", "piece", "picture", "police", "popular", "population", "suggest", "weapon", "water", "window", "woman", "talk", "security", "season", "remain", "relationship", "report" ,"user" ,"result", "this", + "the", "that", "with", "have", "not", "were", "they", "there", "their", "them", "then", "than", "been", "from", "into", "will", "should", "did", "does", + "has", "had", "he", "she", "him", "his", "where", "when", "why", "how", "who", "here", "just", "get", "got", "going", "need", "think", "right", + "never", "always", "maybe", "please", "thank", "thanks", "sorry", "look", "tell", "told", "said", "make", "made", "take", "took", "give", "gave", "back", + "time", "night", "life", "love", "good", "great", "little", "much", "many", "more", "only", "very", "but", "if", "I'm", "it's", "don't", "didn't", "doesn't", + "can't", "isn't", "wasn't", "won't", "we're", "they're", "I've", "you've", "nobody", "before", "after", "years", "closed", "expected", "quiet", }; private static readonly string[] AutoDetectWordsDanish = { "vi", "han", "og", "jeg", "var", "men", "gider", "bliver", "virkelig", "kommer", "tilbage", "Hej", "længere", "gjorde", "dig", "havde", "[Uu]ndskyld", "arbejder", "vidste", "troede", "stadigvæk", "[Mm]åske", "første", "gik", "fortælle", "begyndt", "spørgsmål", "pludselig", "være", "fordi", "været", "sidste","også", "løb", "sagde", "hjem", + "ikke", "til", "hvor", "når", "hvis", "hvem", "hvorfor", "hvordan", "nok", "godt", "fra", "noget", "nogen", "hun", "mig", "deres", + "uden", "mellem", "næste", "anden", "meget", "lidt", "hvad", "nej", "gøre", "gør", "siger", "kender", "kunne", "måtte", "selv", "mere", "andre", "dette", "hele", "sammen", "væk", "penge", "arbejde", "ked", "ærlig", + "fik", "vand", "broen", "vejene", "husene", "senere", "komme", "kommet", + "ventes", "kraftig", "regn", "morgen", "nordlige", "temperaturen", "frysepunktet", "bilister", "forsigtige", }; private static readonly string[] AutoDetectWordsNorwegian = { "vi", "og", "jeg", "var", "men", "igjen", "Nei", "Hei", "noen", "gjøre", "kanskje", "[Tt]renger", "tenker", "skjer", "møte", "veldig", "takk", "penger", "konsept", "hjelp", "forsvunnet", - "skutt", "sterkt", "minste", "første", "fortsette", "inneholder", "gikk", "fortelle", "begynt", "spørsmål", "plutselig" + "skutt", "sterkt", "minste", "første", "fortsette", "inneholder", "gikk", "fortelle", "begynt", "spørsmål", "plutselig", + "ikke", "til", "hvor", "når", "hvis", "hvem", "hvorfor", "hvordan", "også", "nok", "godt", "fra", "noe", "hun", "meg", "deg", "seg", "oss", "deres", + "uten", "mellom", "neste", "annen", "mye", "litt", "nå", "hva", "gjør", "ble", "sier", "kjenner", "kunne", "måtte", + "siste", "selv", "andre", "dette", "hele", "sammen", "vekk", "hjem", "tilbake", "jobb", "jobbet", "lei", "sjanse", + "ærlig", "etter", "aldri", "å", "være", "vært", "hadde", "dere", "fikk", "vann", "brua", "veiene", "husene", "elva", "ventet", "senere", "fjor", + "ventes", "kraftig", "regn", "morgen", "temperaturen", "faller", "løpet", "natten", "bilister", "forsiktige", }; private static readonly string[] AutoDetectWordsSwedish = { "vi", "är", "och", "Jag", "inte", "för", "måste", "Öppna", "Förlåt", "nånting", "ingenting", "jävla", "Varför", "[Ss]nälla", "fattar", "själv", "säger", "öppna", "jävligt", "dörren", "göra", "behöver", "tillbaka", "Varför", "träffa", "kanske", "säga", "hände", "honom", "hennes", "veckor", "tänker", "själv", "pratar", "mycket", "mamma", "dödade", "Ursäkta", "säger", "senaste", "håller", "förstår", "veckan", "varför", "tycker", "första", "gick", - "berätta", "börjat", "människor", "frågor", "fråga", "plötsligt", "skämt" + "berätta", "börjat", "människor", "frågor", "fråga", "plötsligt", "skämt", + "hade", "varit", "att", "ska", "från", "också", "bara", "något", "någon", "någonting", "deras", "gång", "utan", "mellan", "nästa", + "andra", "vad", "tack", "gör", "känner", "kunde", "många", "dessa", "detta", "alla", "allt", "tillsammans", "pengar", "arbete", "ledsen", "chans", "ärlig", + "fick", "vatten", "bron", "vägarna", "husen", "komma", "kommit", "när", "hur", "vem", "vilken", "vilket", "väl", "redan", "ännu", "fortfarande", "sedan", + "innan", "medan", "eftersom", "därför", "stiga", "snabbt", "staden", "stängde", "tyst", "flera", "kullen", "ljug", "prata", "finns", "hon", "henne", + "våra", "vår", "okej", "visst", "precis", "dålig", "dåligt", "bättre", "bäst", "gamla", "unga", "människa", "kvinna", "flicka", "pojke", + "kväll", "vecka", "älskar", "hjälp", "hjälpa", "hejdå", "nån", "nåt", "nånstans", "överallt", "upp", "ner", "kvar", "länge", "ibland", "tidigt", "genast", "verkligen", "nästan", "väldigt", "alldeles", "lika", "annan", "annat", "varje", "båda", "några", "inga", "allting", "vilka", "vart", + "kände", "behövde", "såg", "hörde", "sade", "frågade", "svarade", "tänkte", "försökte", "började", "hjälpte", + "arbetade", "jobbade", "levde", "väntade", "hoppades", "köpte", "betalade", "körde", "träffade", "ringde", "läste", "hittade", "öppnade", + "väntas", "över", "norra", "morgon", "kraftigt", "delarna", "sjunker", "bör", "vara", "försiktiga", "noll", "bord", "två", "självklart", "följ", "menyn", + "först", "utmärkt", }; private static readonly string[] AutoDetectWordsSpanish = @@ -240,7 +540,16 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) { "Buongiorno", "Grazie", "Cosa", "quest[ao]", "quell[ao]", "tutt[io]", "[st]uo", "qualcosa", "ancora", "sono", "bene", "più", "andare", "essere", "venire", "abbiamo", "andiamo", "ragazzi", "signore", "numero", "giorno", "propriamente", "sensitivo", "negativo", "davvero", "faccio", "voglio", "vuole", "perché", "allora", "niente", "anche", "stai", "detto", "fatto", "hanno", "molto", "stato", "siamo", "così", "vuoi", "noi", "vero", "loro", - "fare", "gli", "due" + "fare", "gli", "due", + "è", "sei", "siete", "erano", "stata", "stati", "state", "ho", "hai", "avete", "avevo", "aveva", "avevano", "io", "lui", "lei", "voi", "che", "chi", + "dove", "quanto", "quale", "quali", "qua", "lì", "adesso", "oggi", "domani", "ieri", "nulla", "nessuno", "qualcuno", "tutto", "tutti", "tutta", "tutte", + "troppo", "soltanto", "già", "dopo", "mentre", "quindi", "sì", "prego", "scusa", "scusi", "dispiace", "ciao", "arrivederci", "signora", "signorina", "uomo", + "donna", "ragazzo", "ragazza", "bambino", "bambini", "notte", "mattina", "sera", "anno", "anni", "volta", "amore", "soldi", "presto", "insieme", "peccato", + "forse", "d'accordo", "puoi", "può", "possiamo", "devo", "devi", "deve", "dobbiamo", "vogliamo", "sai", "sappiamo", "pensi", "credo", "credi", "dico", + "dici", "dice", "vedo", "vedi", "vede", "visto", "vieni", "vengo", "vai", "vado", "fai", "facciamo", "parlare", "parlo", "parli", "parla", "lavorare", + "lavoro", "lavoravo", "città", "fiume", "ponte", "acqua", "strade", "collina", "tardi", "mentire", "mentirmi", "possibilità", "onesto", "scorso", + "tranquilla", "rimasta", "fabbrica", "chiudesse", "aspettava", "salisse", "fretta", "sparito", "inondate", "guardava", "portava", "arrivare", "disse", + "averlo", "senti", "dammi", "avuto", }; private static readonly string[] AutoDetectWordsFrench = @@ -254,18 +563,39 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) { "[Nn]ão", "[Ee]ntão", "uma", "ele", "bem", "isso", "você", "sim", "meu", "muito", "estou", "ela", "fazer", "tem", "já", "minha", "tudo", "só", "tenho", "agora", "vou", "seu", "quem", "há", "lhe", "quero", "nós", "são", "ter", "coisa", "dizer", "eles", "pode", "bom", "mesmo", "mim", "estava", "assim", "estão", "até", "quer", "temos", "acho", "obrigado", "também", "tens", - "deus", "quê", "ainda", "noite", "conversar", "Desculpe", "verdade" + "deus", "quê", "ainda", "noite", "conversar", "Desculpe", "verdade", + "é", "foi", "tinha", "têm", "eu", "elas", "vocês", "teu", "tua", "sua", "nosso", "nossa", "isto", "aquilo", "aquele", "aquela", "aqui", "lá", "cá", "hoje", + "amanhã", "ontem", "alguma", "muita", "tão", "apenas", "porquê", "pois", "obrigada", "desculpa", "adeus", "senhor", "senhora", "homem", "mulher", + "rapaz", "menino", "menina", "criança", "crianças", "pessoas", "manhã", "anos", "dinheiro", "sozinho", "coisas", "faço", "fazes", "faz", "fiz", "fez", + "feito", "disse", "diz", "vejo", "vês", "viu", "vem", "vai", "queres", "podes", "preciso", "precisas", "precisamos", "falar", "falo", "falas", "fala", + "falei", "trabalhar", "trabalho", "cidade", "água", "estradas", "ninguém", "subisse", "fechar", "depressa", "devagar", "mas", "onde", "qual", "mentas", + "está bem", "oportunidade", "tiveste", "estiveste", "disse-te", "dá-me", "esperava", "inundadas", "observava", "levava", "disseram", "tinham", "chegar", }; private static readonly string[] AutoDetectWordsGerman = { "und", "auch", "sich", "bin", "hast", "möchte", "müssen", "weiß", "[Vv]ielleicht", "Warum", "jetzt", "Verdammt", "bist", "Darum", "sitzt", "Setz", "Das ist", "Du bist", "nicht", "Scheiße", - "Nein", "für", "gesagt", "zwei", "richtig" + "Nein", "für", "gesagt", "zwei", "richtig", + "ich", "sie", "wir", "ihr", "mich", "dich", "mir", "dir", "uns", "euch", "ihn", "ihm", "ihnen", "mein", "dein", "unser", "dem", "ein", "eine", "einen", + "einem", "einer", "kein", "keine", "nichts", "etwas", "jemand", "niemand", "immer", "niemals", "schon", "dort", "wo", "wohin", "woher", "wann", "wie", "wer", + "weil", "wenn", "dass", "ob", "aber", "oder", "sondern", "denn", "doch", "gut", "schlecht", "sehr", "zu", "von", "aus", "bei", "nach", "über", "unter", + "vor", "hinter", "zwischen", "ohne", "gegen", "durch", "um", "auf", "ist", "sind", "waren", "wird", "werden", "wurde", "habe", "haben", "hatte", "hatten", + "kann", "kannst", "können", "konnte", "muss", "musst", "soll", "sollte", "willst", "wollen", "wollte", "darf", "gehe", "geht", "gehen", "ging", "komme", + "kommt", "kommen", "kam", "mache", "macht", "machen", "gemacht", "sehe", "sieht", "sehen", "gesehen", "weißt", "wissen", "wusste", "denke", "denkst", + "dachte", "sage", "sagt", "sagen", "sagte", "glaube", "lass", "komm", "geh", "schau", "hör", "warte", "frau", "junge", "mädchen", "kinder", "leute", + "menschen", "abend", "zeit", "jahr", "jahre", "haus", "zuhause", "leben", "liebe", "schnell", "langsam", "zusammen", "allein", "schade", "danke", "bitte", + "entschuldigung", "tschüss", "herr", "stadt", "fluss", "brücke", "wasser", "häuser", "straßen", "später", "fabrik", "lange", "gearbeitet", "lüg", "leid", + "reden", "letztes", "ehrlich", "einmal", "ruhig", "jahrelang", "erwartet", "steigen", "hügel", "ganze", "nacht", "nochmal", "hattest", "schloss", }; private static readonly string[] AutoDetectWordsDutch = { - "van", "een", "[Hh]et", "m(ij|ij)", "z(ij|ij)n", "hebben", "alleen", "Waarom", "stripboeken", "Goed", "geweest" + "van", "een", "[Hh]et", "m(ij|ij)", "z(ij|ij)n", "hebben", "alleen", "Waarom", "stripboeken", "geweest", + "niet", "ik", "jij", "wij", "ze", "zij", "hij", "deze", "zo", "zonder", "zijn", "waren", "bent", "wordt", "worden", "werd", "heb", "hebt", "heeft", "hadden", "kun", + "kunnen", "moeten", "moest", "wilt", "willen", "wilde", "zal", "zullen", "zou", "zouden", "gaat", "ging", "komt", "komen", + "kwam", "doe", "doet", "doen", "deed", "gedaan", "zie", "ziet", "zien", "zag", "gezien", "weten", "wist", "denk", "denkt", "dacht", "zeg", "zegt", + "zeggen", "zei", "gezegd", "praten", "mij", "jouw", "hem", "onze", "hun", "niets", "altijd", "misschien", "echt", "veel", "weinig", "slecht", "nieuw", "nacht", "avond", "gisteren", "vandaag", "tijd", "jaren", "thuis", "leven", "mensen", "vrouw", "jongen", "kinderen", "wegen", "huizen", "stijgen", "verwacht", "jarenlang", + "sloot", "liegen", "spijt", "eerlijk", "dank", "bedankt", "alsjeblieft", "tot ziens", }; private static readonly string[] AutoDetectWordsPolish = @@ -275,11 +605,28 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "wiadomości", "Dobrze", "będzie", "Dzień", "przyszłość", "Uratowałaś", "Cześć", "Trzeba", "zginąć", "walczyć", "ludzkość", "maszyny", "Jeszcze", "okrążenie", "wyścigu", "porządku", "detektywie", "przebieralni", "który", "śmierci", "zabić", "wiedźminie", "przeznaczenie", + "gdzie", "kiedy", "dlaczego", "kto", "wy", "mną", "ciebie", "cię", "tobie", "tobą", "jego", "jej", "wam", "są", "był", "była", "było", "byli", "byłeś", + "będę", "będziesz", "jestem", "jesteś", "jesteśmy", "masz", "mamy", "mają", "miał", "miała", "mogę", "możesz", "możemy", "muszę", "musisz", "musimy", "chcę", + "chcesz", "chciałem", "wiesz", "myślę", "myślisz", "mówię", "mówisz", "mówiłem", "mówiła", "mówili", "powiedz", "powiedział", "powiedziała", "widzę", + "widzisz", "widziałem", "chodź", "idź", "idziemy", "zrobić", "zrobiłem", "robisz", "pani", "człowiek", "ludzie", "mężczyzna", "kobieta", "chłopak", + "dziewczyna", "dziecko", "dzieci", "nocy", "rano", "czas", "czasu", "lata", "kocham", "pieniądze", "szybko", "razem", "źle", "bardzo", "tylko", "już", + "teraz", "tutaj", "zawsze", "nigdy", "nikt", "ktoś", "coś", "wszyscy", "ponieważ", "jeśli", "jeżeli", "żeby", "więc", "też", "także", "która", "które", + "całą", "późna", "pracowałem", "okłamuj", "słuchaj", "porozmawiać", "powiedzenia", "szansę", "zeszłym", "roku", "czego", "szczery", "miasto", "spokojne", + "zamknięto", "fabrykę", "rzeka", "mostu", "drogi", "wodą", "patrzyła", "zabiera", "domy", "później", "przewidzieli", "spodziewał", "podniesie", }; private static readonly string[] AutoDetectWordsGreek = { - "μου", "[Εε]ίναι", "αυτό", "Τόμπυ", "καλά", "Ενταξει", "πρεπει", "Λοιπον", "τιποτα", "ξερεις" + "μου", "[Εε]ίναι", "αυτό", "Τόμπυ", "καλά", "Ενταξει", "πρεπει", "Λοιπον", "τιποτα", "ξερεις", + "και", "να", "το", "τα", "την", "τον", "της", "του", "τους", "τις", "με", "σε", "για", "από", "δεν", "μη", "μην", "θα", "ότι", "που", "πως", "πού", "τι", + "γιατί", "πότε", "ποιος", "ποια", "ποιο", "εγώ", "εσύ", "αυτός", "αυτή", "εμείς", "εσείς", "αυτοί", "μας", "σας", "εδώ", "εκεί", "τώρα", "σήμερα", "αύριο", + "χθες", "πάντα", "ποτέ", "κανείς", "κάποιος", "τίποτα", "κάτι", "όλα", "όλοι", "πολύ", "μόνο", "ακόμα", "ήδη", "μετά", "πριν", "επειδή", "αν", "αλλά", "ή", + "ναι", "όχι", "εντάξει", "ευχαριστώ", "παρακαλώ", "συγνώμη", "συγγνώμη", "γεια", "κύριε", "κυρία", "άνθρωπος", "άνθρωποι", "άντρας", "γυναίκα", "αγόρι", + "κορίτσι", "παιδί", "παιδιά", "μέρα", "νύχτα", "πρωί", "χρόνος", "χρόνια", "φορά", "σπίτι", "ζωή", "αγάπη", "λεφτά", "χρήματα", "νεκρός", "γρήγορα", "μαζί", + "μόνος", "είμαι", "είσαι", "είμαστε", "είστε", "ήταν", "ήμουν", "έχω", "έχεις", "έχει", "έχουμε", "είχα", "είχε", "μπορώ", "μπορείς", "μπορεί", "πρέπει", + "θέλω", "θέλεις", "ξέρω", "ξέρεις", "νομίζω", "λέω", "λες", "είπα", "είπε", "βλέπω", "είδα", "έλα", "πάμε", "κάνω", "κάνεις", "κάνει", "έκανα", "δουλειά", + "πόλη", "ποτάμι", "γέφυρα", "νερό", "σπίτια", "αργότερα", "εργοστάσιο", "ήσυχη", "περίμενε", "ανέβαινε", "δρόμοι", "λόφο", "έπαιρνε", "έλεγαν", "δουλεύω", + "ψέματα", "ευκαιρία", "ειλικρινής", "μιλήσουμε", "λυπάμαι", "κοίτα", }; private static readonly string[] AutoDetectWordsRussian = @@ -301,12 +648,18 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "Вона", "мене", "[Нн]і", "[Пп]ривіт", "[Цц]е", "[Щщ]о", "[Йй]ого", "[Вв]ін", "[Яя]к", "[Гг]аразд", "[Яя]кщо", "[Мм]ені", "[Тт]вій", "[Її]х", "[Вв]ітаю", "[Дд]якую", "вже", "було", "був", "цього", "нічого", "немає", "може", "знову", "бо", "щось", "щоб", "цим", "тобі", "хотів", "твоїх", "мої", "мій", "має", "їм", "йому", "дуже", "Привіт", "побачення", "тобою", "побачення", "Гаразд", "повинні", + "ти", "ми", "ви", "вони", "він", "зараз", "сьогодні", "вчора", "завжди", "ніколи", "ніхто", "хтось", "усе", "всі", "тільки", + "лише", "ні", "погано", "будь ласка", "вибач", "вибачте", "пробач", "пан", "пані", "людина", "чоловік", "жінка", "хлопець", + "дівчина", "дитина", "діти", "ніч", "ранок", "рік", "роки", "дім", "додому", "життя", "кохаю", "гроші", "швидко", "повільно", + "разом", "шкода", "маю", "маєш", "можу", "можеш", "мусимо", "хочеш", "хоче", "хочемо", "знаєш", "кажу", + "кажеш", "сказав", "бачу", "бачиш", "бачив", "іди", "ходімо", "зробити", "зробив", "робиш", "місто", "роками", "тихим", "доки", + "закрився", "очікував", "річка", "підніметься", "ранку", "міст", "зник", "водою", "дивилася", "забирає", "будинки", "пізніше", "казали", + "передбачали", "усю", "казав", "працював", "пізна", "бреши", "сказати", "мав", "свій", "минулого", "чого", "чесним", }; private static readonly string[] AutoDetectWordsAlbanian = { - "është", "këtë", "Unë", "mirë", "shumë", "Çfarë", "çfarë", "duhet", "Është", "mbrapa", "Faleminderit", "vërtet", "diçka", "gjithashtu", "gjithe", "eshte", "shume", "vetem", "tënde", - "çmendur", "vullnetin", "vdekur" + "është", "të", "një", "për", "nuk", "më", "që", "dhe", "në", "ku", "kush", "pse", "çfarë", "shumë", "mirë", "keq", "faleminderit", "zotëri", "zonjë", "këtu", "atje", "tani", "nesër", "dje", "gjithë", "gjithmonë", "kurrë", "asgjë", "dikush", "diçka", "ishte", "isha", "jemi", "janë", "kemi", "kanë", "kisha", "kishte", "duhet", "dua", "doja", "tha", "thuaj", "thashë", "shiko", "eja", "shko", "bëj", "bëri", "njeri", "njerëz", "burrë", "grua", "djalë", "vajzë", "fëmijë", "ditë", "natë", "natën", "kohë", "vitin", "herë", "shtëpi", "jetë", "dashuri", "edhe", "ose", "nga", "sepse", "nëse", "kështu", "ashtu", "vetëm", "gjë", "gjëra", "lutem", "falni", "kuptoj", "mendoj", "besoj", "shoh", "vij", "unë", "ata", "ato", "ajo", "këtë", "mbrapa", "vërtet", "gjithashtu", "tënde", "çmendur", "vullnetin", "vdekur", "qenë", "punoja", "deri", "vonë", "gënje", "vjen", "rregull", "flasim", "thënë", "jep", "shans", "pate", "kaluar", "sinqertë", "qetë", "lumi", "ura", "rrugët", "uji", "shtëpitë", "vonë", "qyteti", }; private static readonly string[] AutoDetectWordsArabic = @@ -316,7 +669,15 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) private static readonly string[] AutoDetectWordsFarsi = { - "این", "برای", "اون", "سیب", "کال", "رو", "خيلي", "آره", "بود", "اون", "نيست", "اينجا", "باشه", "سلام", "ميکني", "داري", "چيزي", "چرا", "خوبه" + "این", "برای", "اون", "سیب", "کال", "رو", "خيلي", "آره", "بود", "اون", "نيست", "اينجا", "باشه", "سلام", "ميکني", "داري", "چيزي", "چرا", "خوبه", + "که", "را", "به", "از", "با", "در", "است", "هست", "نیست", "نبود", "باشد", "تو", "او", "شما", "آنها", "اونها", "اینجا", "آنجا", "اونجا", "حالا", "الان", + "امروز", "فردا", "دیروز", "همیشه", "هیچوقت", "هیچ", "چیزی", "کسی", "همه", "خیلی", "هنوز", "دیگه", "دیگر", "چی", "چه", "کجا", "کی", "چطور", "چقدر", + "کدام", "بله", "نه", "خوب", "بد", "ممنون", "مرسی", "متشکرم", "لطفا", "لطفاً", "ببخشید", "متأسفم", "متاسفم", "خداحافظ", "آقا", "خانم", "مرد", "زن", "پسر", + "دختر", "بچه", "مردم", "روز", "شب", "سال", "بار", "خانه", "خونه", "زندگی", "پول", "مرده", "زود", "دیر", "تنها", "درست", "البته", "واقعا", "واقعاً", "باید", + "نباید", "میدونم", "می‌دونم", "نمیدونم", "نمی‌دونم", "می‌دانم", "نمی‌دانم", "فکر", "می‌کنم", "میکنم", "می‌کنی", "میکنی", "می‌کردم", "می‌خواهم", "میخوام", + "می‌خوام", "می‌خواهی", "می‌خواهد", "می‌تونم", "می‌توانم", "نمی‌تونم", "گفتم", "گفت", "گفتی", "بگو", "ببین", "بیا", "برو", "کرد", "کردم", "کردی", "کار", + "شهر", "تا", "یک", "یه", "دو", "داری", "دارم", "داره", "می‌کنیم", "بودی", "بودم", "بودند", "دروغ", "نگو", "حرف", "بزنیم", "فرصت", "بده", "پارسال", "صادق", + "آرام", "کارخانه", "بسته", "رودخانه", "بالا", "بیاید", "جاده‌ها", "آب", "خانه‌ها", "تماشا", "می‌گفتند", "دیده", "آمدنش", }; private static readonly string[] AutoDetectWordsHebrew = @@ -324,11 +685,26 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "אתה", "אולי", "הוא", "בסדר", "יודע", "טוב", "אֶת", "שֶׁל", "עַל", "הוּא", "אֲשֶׁר", "הִיא", "הַמְלֶט", "נָסִיךְ", "הֲלֹא", "עוֹד", "אֵין", "אֲנִי", "זֹאת", "וְלֹא", "כָּךְ", "הִנֵּה", "מְאֹד", "אֲדוֹנִי", + "אני", "את", "היא", "אנחנו", "אתם", "הם", "הן", "זה", "זאת", "של", "על", "עם", "לא", "כן", "מה", "מי", "איפה", "למה", "מתי", "איך", "כמה", "יש", "אין", + "היה", "הייתי", "היית", "יהיה", "רוצה", "יכול", "צריך", "חושב", "אומר", "אמרתי", "אמרת", "אמר", "רואה", "ראיתי", "בוא", "לך", "תן", "עשה", "עשיתי", "לעשות", + "לדבר", "עכשיו", "היום", "מחר", "אתמול", "תמיד", "אף פעם", "כלום", "משהו", "מישהו", "הכל", "כל", "רק", "גם", "עוד", "כבר", "מאוד", "רע", "תודה", "בבקשה", + "סליחה", "שלום", "אדוני", "גבירתי", "איש", "אנשים", "גבר", "אישה", "ילד", "ילדה", "ילדים", "יום", "לילה", "בוקר", "זמן", "שנה", "שנים", "פעם", "בית", "חיים", + "אהבה", "כסף", "מת", "מהר", "ביחד", "לבד", "חבל", "אבל", "כי", "אם", "או", "אז", "שם", "פה", "כאן", "הזדמנות", "לשקר", "תשקר", "עבדתי", "מאוחר", "לומר", + "מצטער", "להיות", "העיר", "שקטה", "המפעל", "נסגר", "הנהר", "יעלה", "הגשר", "נעלם", "הכבישים", "הוצפו", "הבתים", "המים", "הגבעה", "צפתה", "אמרו", "ראו", + "מגיע", "ממני", "ממך", "לי", "לו", "לה", "לנו", "לכם", "להם", "אותי", "אותך", "אותו", "אותה", "אותנו", "שלי", "שלך", "שלו", "שלה", "שלנו", "הלילה", "תראה", + "צריכים", "שעברה", "פעם אחת", "איש לא", "ציפה", "הייתה", }; private static readonly string[] AutoDetectWordsVietnamese = { - "không", "[Tt]ôi", "anh", "đó", "ông", "tôi", "phải", "người", "được", "Cậu", "chúng", "chuyện", "muốn", "những", "nhiều" + "không", "[Tt]ôi", "anh", "đó", "ông", "tôi", "phải", "người", "được", "Cậu", "chúng", "chuyện", "muốn", "những", "nhiều", + "của", "và", "có", "một", "này", "cho", "với", "để", "cô", "bà", "chị", "nó", "họ", "mình", "gì", "sao", "đâu", "nào", "bao", "khi", "lúc", "giờ", "luôn", + "chưa", "đã", "sẽ", "đang", "rồi", "nữa", "cũng", "chỉ", "rất", "quá", "lắm", "vâng", "dạ", "đúng", "sai", "tốt", "xấu", "chào", "đi", "đến", "về", "làm", + "nói", "biết", "nghĩ", "thấy", "nhìn", "nghe", "cần", "thích", "yêu", "sống", "chết", "tiền", "nhà", "năm", "ngày", "đêm", "sáng", "tối", "lần", "nhanh", + "cùng", "nhau", "thật", "chắc", "thế", "vậy", "nhưng", "vì", "nếu", "hoặc", "đây", "đấy", "kia", "tại", "trong", "ngoài", "trên", "dưới", "trước", "giữa", + "sông", "cầu", "đường", "nước", "đồi", "muộn", "dối", "cảm ơn", "xin lỗi", "làm ơn", "có thể", "bây giờ", "hôm nay", "ngày mai", "hôm qua", "thị trấn", + "nhà máy", "đóng cửa", "yên tĩnh", "cơ hội", "thành thật", "chuyện này", "ở đâu", "nghe này", "còn", "ở", "ai đó", "chúng ta", "mọi người", "nhiều năm", + "trước khi", "sau đó", }; private static readonly string[] AutoDetectWordsHungarian = @@ -336,7 +712,18 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "hogy", "lesz", "tudom", "vagy", "mondtam", "még", "vagyok", "csak", "Hát", "felesége", "Csak", "utána", "jött", "Miért", "Akkor", "magát", "holnap", "Tudja", "Köszönöm", "élet", "Örvendek", "vissza", "hogy", "tudom", "Rendben", "Istenem", "Gyerünk", "értem", "vagyok", "hiszem", "történt", "rendben", "olyan", "őket", "vannak", "mindig", "Kérlek", - "Gyere", "kicsim", "vagyunk" + "Gyere", "kicsim", "vagyunk", + "nem", "és", "egy", "ez", "az", "volt", "nincs", "már", "itt", "ott", "mit", "hol", "ki", "én", "ő", "ők", "nekem", "neked", "neki", "engem", "velem", + "veled", "tudod", "akarom", "akarok", "kell", "lehet", "jó", "rossz", "kérem", "bocsánat", "sajnálom", "szia", "uram", "asszonyom", "ember", "emberek", + "férfi", "nő", "fiú", "lány", "gyerek", "éjjel", "éjszaka", "reggel", "idő", "év", "ház", "haza", "szeretlek", "pénz", "gyorsan", "lassan", "együtt", + "egyedül", "kár", "soha", "senki", "valaki", "semmi", "valami", "minden", "mindenki", "nagyon", "talán", "persze", "tényleg", "igazán", "biztos", "azt", + "ezt", "ezek", "azok", "sincs", "mert", "aki", "ami", "amit", "ide", "oda", "innen", "onnan", "után", "előtt", "alatt", "között", "nélkül", "miatt", + "szerint", "mondta", "mondd", "látom", "látod", "láttam", "menj", "menjünk", "csinálni", "csináltam", "csinálsz", "beszélnünk", "beszélni", "hazudj", + "esélyt", "esélyed", "tavaly", "őszinte", "város", "évekig", "csendes", "mielőtt", "gyár", "bezárt", "számított", "folyó", "megárad", "reggelre", "híd", + "eltűnt", "utak", "víz", "dombról", "nézte", "ahogy", "elviszi", "házakat", "később", "mondták", "látták", "előre", "voltál", "egész", "sokáig", "dolgoztam", + "nézd", "tőlem", "legyél", "egyszer", + "van", "estét", "asztaluk", "két", "természetesen", "kövessenek", "először", "étlapot", "hal", "ma", "este", "kiváló", "északon", "heves", "esőzés", + "várható", "hőmérséklet", "fagypont", "alá", "süllyed", "autósok", "legyenek", "különösen", "óvatosak", "utakon", }; private static readonly string[] AutoDetectWordsTurkish = @@ -356,7 +743,12 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) private static readonly string[] AutoDetectWordsCroatianAndSerbian = { - "Hajde", "znam", "ćemo", "šta", "sam", "ali", "nije", "Nije", "samo", "ovo", "kako", "dobro", "Dobro", "sve", "tako", "će", "mogu", "ću", "zašto", "nešto", "za", "misliš", "možeš", "možemo", "ništa", "znaš", "ćemo", "znam" + "Hajde", "znam", "ćemo", "šta", "sam", "ali", "nije", "Nije", "samo", "ovo", "kako", "dobro", "Dobro", "sve", "tako", "će", "mogu", "ću", "zašto", "nešto", "za", "misliš", "možeš", "možemo", "ništa", "znaš", "ćemo", "znam", + "li", "ga", "mu", "joj", "vam", "ih", "ili", "kad", "koliko", "kao", "još", "već", "sada", "tamo", "danas", "sutra", "nikad", "nikada", "svi", "svaki", + "vrlo", "puno", "mnogo", "više", "manje", "loše", "hvala", "molim", "izvoli", "gospodin", "gospođa", "ljudi", "muškarac", "noć", "godina", "godine", "kuća", + "kući", "ljubav", "novac", "mrtav", "zajedno", "šteta", "imam", "imaš", "ima", "imamo", "imaju", "može", "moraš", "moramo", "hoću", "hoćeš", "hoće", "želim", + "želiš", "zna", "kažem", "kažeš", "rekao", "rekla", "reci", "vidim", "vidiš", "dođi", "idi", "idemo", "raditi", "radiš", "večeras", "jelovnik", "prvo", + "naravno", "očekuje", "jaka", "kiša", "noći", "ispod", "nule", "vozači", "trebali", "posebno", "oprezni", "pasti", "dvoje", "temperature", }; private static readonly string[] AutoDetectWordsCroatian = @@ -365,18 +757,28 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "poslije", "događa", "vjerovati", "vjerojatno", "vjerujem", "točno", "razumijem", "vidjela", "cijeli", "svijet", "obitelj", "volio", "sretan", "dovraga", "svijetu", "htjela", "vidjeli", "negdje", "želio", "ponovno", "djevojka", "umrijeti", "čovjeka", "mjesta", "djeca", "osjećam", "uopće", "djecu", "naprijed", "obitelji", "doista", "mjestu", "lijepa", "također", "riječ", "tijelo", "vrijeme", "mlijeko", "lijepo", + "stol", "slijedite", "izvrsna", "sjeveru", "tijekom", "cestama", "jučer", "tisuću", "tjedan", "mjesec", "svjetlo", "glazba", "zrak", "kruh", "tvrtka", + "siječanj", "vjetar", "snijeg", "rijeka", "cijena", "vrijedi", "pješice", "sljedeći", "prijatelj", "prijatelji", }; private static readonly string[] AutoDetectWordsSerbian = { "šta", "ovde", "gde", "ko", "pre", "uvek", "vreme", "videti", "neko", "video", "niko", "ćao", "lepo", "izvini", "hteo", "mesto", "izvinite", "čovek", "dole", "čoveče", "dve", "dete", "deo", "posle", "dešava", "verovati", "verovatno", "verujem", "tačno", "razumem", "videla", "ceo", "svet", "porodica", "voleo", "srećan", "dođavola", "svetu", "htela", "videli", "negde", "želeo", "ponovo", "devojka", "umreti", "čoveka", "mesta", "deca", "osećam", - "uopšte", "decu", "napred", "porodicu", "zaista", "mestu", "lepa", "takođe", "reč", "telo" + "uopšte", "decu", "napred", "porodicu", "zaista", "mestu", "lepa", "takođe", "reč", "telo", + "sto", "veče", "pratite", "odlična", "severu", "tokom", "putevima", "juče", "hiljadu", "nedelja", "mesec", "svetlo", "muzika", "vazduh", "hleb", "firma", + "januar", "trebalo", "budu", "vetar", "sneg", "reka", "cena", "vredi", "peške", "sledeći", "prijatelj", "prijatelji", }; private static readonly string[] AutoDetectWordsSerbianCyrillic = { - "ће", "ћемо", "Хоћу", "сам", "али", "није", "само", "ово", "како", "добро", "све", "тако", "ће", "могу", "ћу", "зашто", "нешто", "за", "шта", "овде", "бити", "чини", "учениче", "побегне", "остати", "Један", "Назад", "Молим" + "ће", "ћемо", "Хоћу", "сам", "али", "није", "само", "ово", "како", "добро", "све", "тако", "ће", "могу", "ћу", "зашто", "нешто", "за", "шта", "овде", "бити", "чини", "учениче", "побегне", "остати", "Један", "Назад", "Молим", + "сутра", "северу", "очекује", "јака", "киша", "температуре", "ноћи", "испод", "нуле", "возачи", "требало", "посебно", "опрезни", "путевима", "вече", "двоје", + "наравно", "пратите", "можемо", "прво", "јеловник", "вечерас", "одлична", "сада", "данас", "јуче", "увек", "никад", "нико", "неко", "ништа", "сви", "више", + "хвала", "људи", "дан", "ноћ", "кућа", "љубав", "новац", "заједно", "хоћеш", "хоће", "желим", "кажем", "рекао", "рекла", "дођи", "идемо", "хајде", "радити", + "радио", "лажеш", "жао", "реду", "морамо", "разговарамо", "каже", "прилику", "своју", "прошле", "једном", "будеш", "искрен", "годинама", "миран", + "затворила", "нарасти", "јутра", "нестао", "поплављени", "брда", "односи", "куће", "касније", "ја", "је", "су", "смо", "сте", "они", "она", "мене", "тебе", + "њега", "њу", "нас", "вас", "их", "ми", "ти", "ви", "ко", "где", "кад", "јер", "ако", "или", "па", "још", "већ", "много", "мало", "лоше", }; private static readonly string[] AutoDetectWordsSerbianCyrillicOnly = @@ -386,7 +788,16 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) private static readonly string[] AutoDetectWordsIndonesian = { - "yang", "tahu", "bisa", "akan", "tahun", "tapi", "dengan", "untuk", "rumah", "dalam", "sudah", "bertemu" + "yang", "tahu", "bisa", "akan", "tahun", "tapi", "dengan", "untuk", "rumah", "dalam", "sudah", "bertemu", + "aku", "kamu", "kita", "mereka", "saya", "anda", "ini", "itu", "dari", "pada", "oleh", "atau", "tetapi", "karena", "kalau", "jika", "supaya", "agar", "apa", + "siapa", "mana", "kapan", "mengapa", "kenapa", "bagaimana", "berapa", "tidak", "bukan", "belum", "jangan", "ada", "adalah", "harus", "mau", "ingin", "boleh", + "dapat", "sedang", "masih", "lagi", "juga", "saja", "hanya", "sangat", "sekali", "sekarang", "nanti", "tadi", "kemarin", "besok", "hari", "malam", "pagi", + "waktu", "kali", "orang", "pria", "wanita", "anak", "ibu", "ayah", "teman", "hidup", "cinta", "uang", "mati", "cepat", "pelan", "bersama", "sendiri", + "sayang", "terima kasih", "maaf", "tolong", "selamat", "pak", "bu", "tuan", "nyonya", "semalaman", "kerja", "larut", "bohong", "dengar", "bicara", "soal", + "perlu", "dikatakan", "beri", "kesempatan", "lalu", "jujur", "kota", "sunyi", "pabrik", "ditutup", "menyangka", "sungai", "naik", "begitu", "menjelang", + "jembatan", "hilang", "jalan", "terendam", "menyaksikan", "bukit", "menyapu", "belakangan", "berkata", "menduganya", "kubilang", "padaku", "dariku", "oke", + "hujan", "turun", "utara", "diperkirakan", "suhu", "bawah", "nol", "pengemudi", "hati-hati", "ekstra", "meja", "tentu", "silakan", "ikuti", "lihat", + "menunya", "dulu", "ikannya", "enak", "deras", }; // Catalan — distinctive words that don't collide with Spanish/French/Italian/Portuguese @@ -396,8 +807,15 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "què", "això", "amb", "però", "molt", "moltes", "molts", "gràcies", "aquest", "aquesta", "aquestes", "nosaltres", "vosaltres", "vull", "puc", "pots", "sóc", "ets", "són", "fer", "fas", "fem", "tinc", "tens", "seva", "teu", "meu", "ara", "res", "també", "perquè", "sisplau", "endavant", "ningú", - "sempre", "doncs", "gairebé", "cadascú" - }; + "sempre", "doncs", "gairebé", "cadascú", + "estat", "estic", "estàs", "està", "estem", "estan", "sou", "és", "eren", "heu", "havia", "té", "tenim", "tenen", "tenia", "vols", "volem", + "podem", "cal", "saps", "sap", "crec", "creus", "dius", "diu", "vaig", "vam", "anar", "feu", "fet", "veig", "veus", "veure", "vine", "vinc", + "noia", "nen", "nit", "matí", "anys", "vegada", "ràpid", "perdó", "d'acord", "adéu", "senyor", + "senyora", "meva", "teva", "nostre", "vostre", "ells", "ell", "vostè", "ens", "allà", "allò", "més", "tampoc", "després", + "abans", "dins", "algú", "treballar", "treballava", "treballo", "mentis", "greu", "parlar", "oportunitat", "sincer", "poble", + "fàbrica", "tanqués", "riu", "carreteres", "turó", "aigua", "vist", "tothom", "encara", "només", "potser", "si us plau", "fer-ho", + "demà", "dels", "pels", "pel", "s'esperen", "baixaran", "d'anar", "pluges", "bona", "teniu", "taula", "seguiu-me", "podríem", "primer", "peix", "excel·lent", + "temperatures", "conductors", }; // Tagalog / Filipino — distinctive Austronesian function words; low overlap with European // languages. Spanish loanwords ("para", "pero") are intentionally left out to avoid collisions. @@ -406,7 +824,15 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "ang", "ng", "mga", "ako", "ikaw", "siya", "kami", "tayo", "kayo", "sila", "ito", "iyan", "iyon", "dito", "diyan", "doon", "hindi", "wala", "mayroon", "meron", "salamat", "kumusta", "bakit", "ano", "sino", "saan", "kailan", "paano", "gusto", "alam", "ginagawa", "talaga", "naman", "kasi", "namin", - "natin", "ninyo", "nila", "niya", "kaya", "dahil", "ganito", "ganyan", "kung", "kapag", "mahal" + "natin", "ninyo", "nila", "niya", "kaya", "dahil", "ganito", "ganyan", "kung", "kapag", "mahal", + "ka", "rin", "daw", "nga", "pala", "yung", "yan", "yun", "oo", "opo", "sige", "pwede", "puwede", "huwag", "wag", "walang", "maraming", "marami", "konti", + "lahat", "kahit", "tungkol", "ngayon", "bukas", "kahapon", "gabi", "umaga", "araw", "oras", "taon", "beses", "bahay", "buhay", "patay", "mabilis", "mabagal", + "magkasama", "mag-isa", "totoo", "sigurado", "siguro", "mabuti", "masama", "maganda", "malaki", "maliit", "matanda", "bata", "tao", "lalaki", "babae", + "kaibigan", "sabi", "sinabi", "sabihin", "gawin", "ginawa", "ayaw", "akala", "tingin", "nakita", "halika", "tara", "pumunta", "nagpunta", "umuwi", "trabaho", + "pasensya", "pakiusap", "bigyan", "pagkakataon", "nagkaroon", "nakaraang", "maging", "tapat", "minsan", "tahimik", "bayan", "loob", "bago", "magsara", + "pabrika", "tataas", "ilog", "ganoon", "kabilis", "pagsapit", "tulay", "binaha", "kalsada", "pinagmasdan", "burol", "tinangay", "tubig", "kalaunan", + "darating", "buong", "kailangan", "sa'yo", "akong", "nating", "masasabi", "pag-usapan", "pagsinungalingan", "nag-overtime", "tingnan", "okay lang", "noong", + "mula", }; // Afrikaans — only words that are NOT also valid Dutch, so a Dutch clip is never mistaken @@ -415,8 +841,11 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) { "nie", "baie", "jy", "jou", "ek", "moenie", "dankie", "asseblief", "hoekom", "hierdie", "daardie", "sal", "vir", "hulle", "saam", "gesels", "meisie", "altyd", "regtig", "aandag", "vinnig", "seblief", - "nogal", "sommer", "elkeen" - }; + "nogal", "sommer", "elkeen", + "julle", "sy", "hy", "'n", "oukei", "verlede", "gesê", "sê", "gewerk", "gesluit", "gekyk", "dink", "glo", "gepraat", "gemaak", "gedoen", "gekom", "gegaan", + "gewees", "geword", "gevind", "gekry", "gegee", "gesien", "geneem", "verwag", "stadig", "almal", "seker", "miskien", "sou", "moes", "wou", "jare", "nag", + "môre", "vandag", "gister", "tyd", "lewe", "mense", "vrou", "seun", "kinders", "huise", "mekaar", "styg", "oggend", "paaie", "lank", "terwyl", "sodat", + "eers", "laaste", "homself", "hul", "sonder", "binne", "buite", "hieroor", "wegneem", "eerlik", }; private static readonly string[] AutoDetectWordsThai = { @@ -430,26 +859,54 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "있어요", "보여줘", "괜찮아요", "목소릴", "기억이", "저주를", "좋아요" }; + // Everyday Macedonian words, preferring the ones Bulgarian and Serbian spell + // differently (ќе/ще/ће, јас/аз/ја, тоа/това/то, зошто/защо/зашто, ...). The old list + // was mostly words from one space film plus OCR misreads with Latin letters in them. private static readonly string[] AutoDetectWordsMacedonian = { - "господине", "Нема", "господине", "работа", "вселената", "Може", "треба", "Треба", "слетување", "капсулата", "време", "Френдшип", "Прием", "Добро", "пресметки", "Благодарам", "нешто", "Благодарам", "орбитата", "инженер", - "Харисон", "Фала", "тоалет", "орбита", "знаеме", "Супервизор", "жени", "Добра", "требаат", "што", "дeкa", "eшe", "кучe", "Руиз", "кучeто", "кучињa", "Бјути", "имa", "многу", "кучињaтa", "AДЗЖ", "Животни", "моЖe", "мaчe", - "мecто", "имaмe", "мaчињa", "пpвото", "пpaвaт", "нeшто", "колку", - "што", "тоа", "дека", "Што", "Дали", "треба", "беше", "таму" + "ќе", "што", "тоа", "дека", "дали", "јас", "тој", "таа", "тие", "сум", "треба", "требаат", "нема", "може", "можеби", "мора", + "многу", "нешто", "ништо", "некој", "никој", "сите", "колку", "каде", "зошто", "затоа", "тука", "таму", "уште", "веќе", + "сега", "денес", "секогаш", "никогаш", "сакам", "мислам", "знаеме", "одиме", "правиш", "знаеш", "имаш", "беше", "време", + "работа", "луѓе", "жени", "добро", "добра", "благодарам", "фала", "господине", "тоалет", "инженер", "вселената", "орбитата", + "очекуваат", "дождови", "температурите", "паднат", "нулата", "текот", "ноќта", "возачите", "бидат", "двајца", "секако", "следете", "видиме", "менито", + "вечерва", "повеќе", "молам", "госпоѓо", "ноќ", "куќа", "љубов", "заедно", "можам", "сакаш", "велам", "дојди", "ајде", "работев", "зборуваме", "својата", + "минатата", "шанса", "еднаш", "бидеш", "градот", "мирен", "фабриката", "реката", "издигне", "утрото", "мостот", "патиштата", "поплавени", "ридот", "водата", + "куќите", "подоцна", "велеа", "доаѓа", "утре", "дожд", "ветер", "снег", "цена", "пријател", "пријатели", "следниот", "недела", "месец", "илјада", "јануари", + "музика", "воздух", "леб", "фирма", "пешки", "вреди", "светло", }; private static readonly string[] AutoDetectWordsFinnish = { "että", "kuin", "minä", "mitään", "Mutta", "siitä", "täällä", "poika", "Kiitos", "enää", "vielä", "tässä", "sulkeutuu", "Soitetaan", "Soita", "Onnistui", "Mitä", "Etuovi", "tippiä", "antaa", "Onko", "Hidasta", "tuntia", "tilata", "päästä", "palolaitokselle", "hätänumeroon", "aikaa", "Tämä", "Sinulla", "Palauttaa", "Kiitos", "Arvostele", "Älä", "toimi", "televisiota", "takaisin", "reitin", "pitäisi", "palauttaa", "nopeamman", "mitään", - "meidät", "maksaa", "kuullut", "kaikki", "jälkeen", "ihmiset", "hätäkeskukseen", "hiljaa", "haluat", "enää", "enemmän", "auttaa", "Tunkeilijahälytys", "Pysähdy", "Princen", "Käänny", "Kyllä", "Kiva", "Haluatko", "Haluan" + "meidät", "maksaa", "kuullut", "kaikki", "jälkeen", "ihmiset", "hätäkeskukseen", "hiljaa", "haluat", "enää", "enemmän", "auttaa", "Tunkeilijahälytys", "Pysähdy", "Princen", "Käänny", "Kyllä", "Kiva", "Haluatko", "Haluan", + "ei", "ole", "olen", "olet", "oli", "ollut", "olemme", "ovat", "olla", "olisi", "ennen", "jos", "vaan", "vai", "tai", "myös", "vain", "nyt", "sitten", + "tuolla", "siellä", "sinne", "tänne", "hän", "hänen", "häntä", "minun", "minua", "minulle", "sinun", "sinua", "sinulle", "sinä", "meidän", "meille", + "teidän", "heidän", "joku", "jotain", "mikään", "kukaan", "kaikkea", "paljon", "hyvin", "hyvä", "hyvää", "huomenta", "anteeksi", "totta", "ehkä", "varmasti", + "tietenkin", "koskaan", "aina", "ikinä", "juuri", "ihan", "niin", "näin", "sillä", "siksi", "koska", "vaikka", "kuitenkin", "eikä", "eivät", "emme", "voi", + "voin", "voit", "voimme", "voisi", "täytyy", "pitää", "tiedän", "tiedät", "tiedä", "luulen", "luulin", "tarkoitan", "ymmärrän", "tule", "tulee", "menee", "tehdä", "tein", "teit", "tehnyt", "sanoi", "sanoin", "sanoit", "sanoa", "nähdä", "näit", "nähnyt", "katso", "odota", "ota", "mies", + "nainen", "tyttö", "lapsi", "ihminen", "päivä", "päivää", "yö", "yöllä", "aamu", "aamulla", "aika", "vuosi", "vuotta", "vuoden", "kerran", "kertaa", "koko", + "kotiin", "koti", "elämä", "rakastan", "kuollut", "tapahtui", "tapahtunut", "tarvitsen", "apua", "rahaa", "hetki", "hetkinen", "nopeasti", "yhdessä", + "yksin", "miksi", "missä", "miten", "milloin", "kuka", "joka", "jotka", "mihin", "silta", "joen", "kaupunki", "vesi", "talot", "tiet", "myöhemmin", "töissä", + "myöhään", "valehtele", "mahdollisuus", "rehellinen", + "huomenna", "eilen", "tänään", "odotetaan", "pohjoisessa", "pohjoiseen", "etelässä", "sataa", "sadetta", "rankkasadetta", "lämpötila", "laskee", "pakkasen", + "puolelle", "autoilijoiden", "kannattaa", "erityisen", "varovaisia", "iltaa", "teillä", "pöytää", "kahdelle", "tietysti", "seuratkaa", "voisimmeko", + "ruokalistan", "ensin", "kala", "erinomaista", }; private static readonly string[] AutoDetectWordsRomanian = { "pentru", "oamenii", "decât", "[Vv]reau", "[Ss]înt", "Asteaptã", "Fãrã", "aici", "domnule", "trãiascã", "niciodatã", "înseamnã", "vorbesti", "fãcut", "spune", "făcut", "când", "aici", "Asta", "văzut", "dacă", "câteva", "amândoi", "Când", "totuși", "mașină", "aceeași", "întâmplat", "niște", "ziua", "noastră", "cunoscut", "decat", "[Tt]rebuie", "[Aa]cum", "Poate", "vrea", "soare", "nevoie", "daca", "echilibrul", "vorbesti", "zeului", - "atunci", "memoria", "soarele" + "atunci", "memoria", "soarele", + "ești", "sunt", "suntem", "sunteți", "erau", "fost", "fie", "și", "sau", "dar", "că", "în", "despre", "până", "după", "înainte", "fără", "între", "peste", + "unde", "cine", "cât", "câte", "acolo", "azi", "astăzi", "mâine", "mereu", "niciodată", "nimic", "nimeni", "cineva", "ceva", "toți", "toate", "toată", + "foarte", "mult", "prea", "doar", "numai", "încă", "bine", "rău", "mulțumesc", "mersi", "scuze", "doamnă", "domnișoară", "ea", "mie", "ție", "tine", "nouă", + "lor", "mea", "tău", "său", "nostru", "vostru", "vrei", "vrem", "poți", "putem", "știu", "știi", "știe", "cred", "crezi", "spun", "spui", "spus", "zic", + "zis", "văd", "vezi", "haide", "fac", "faci", "oameni", "bărbat", "femeie", "băiat", "fată", "copil", "copii", "zi", "noapte", "noaptea", "dimineață", + "timp", "ani", "anul", "acasă", "casă", "viață", "viața", "dragoste", "bani", "repede", "încet", "împreună", "singur", "păcat", "sigur", "desigur", "chiar", + "deci", "așa", "astfel", "iar", "lucru", "lucrat", "lucrez", "târziu", "devreme", "orașul", "fabrica", "râul", "podul", "drumurile", "inundate", "privea", + "apa", "casele", "văzuseră", "venind", "minți", "vorbim", "șansă", "trecut", "măcar", "dată", "ai fost", }; // Czech and Slovak languages have many common words (especially when non flexed) @@ -462,14 +919,26 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) private static readonly string[] AutoDetectWordsCzech = { ".*[Řř].*", ".*[ůě].*", "[Bb]ýt", "[Jj]sem", "[Jj]si", "[Jj]á", "[Mm]ít", "[Aa]no", "[Nn]e", "[Nn]ic", "[Dd]en", "[Jj]en", "[Cc]o", "[Jj]ak[o]?", "[Nn]ebo", "[Pp]ři", "[Pp]ro", "[Pp]řed.*", "[Jj](ít|du|de|deme|dou)", - "[Mm]ezi", "[Jj]eště", "[Čč]lověk", "[Pp]odle", "[Dd]alší", "přejděte", "používat", "obrázky", "obrázek", "vašem", "vygenerovat" - }; + "[Mm]ezi", "[Jj]eště", "[Čč]lověk", "[Pp]odle", "[Dd]alší", "přejděte", "používat", "obrázky", "obrázek", "vašem", "vygenerovat", + "nikdo", "někdo", "něco", "všechno", "teď", "proč", "kdo", "kdy", "včera", "zítra", "děkuji", "promiň", "paní", "lidé", "peníze", "rychle", "pomalu", "řekl", + "řekla", "viděl", "udělat", "město", "řeka", "později", "viděli", "chci", "vím", "víš", "může", "můžu", "můžeš", "jsou", "byl", "byla", "bylo", "jsme", "jste", "nejsem", "jenom", "velmi", "moc", "dobře", "špatně", "pán", "dítě", "děti", "dívka", "holka", "kluk", "ráno", "domů", "miluji", "miluju", + "mají", "měl", "měla", "říkám", "říkal", "říkali", "řekni", "pojď", "jdi", "jdeme", "dělat", "dělám", "děláš", "udělal", "nevím", "nechci", "nemůžu", + "nemám", "potřebuju", "potřebuji", "jasně", "samozřejmě", "opravdu", "vážně", "tady", "pak", "protože", "jestli", "pokud", "nelži", "promluvit", "šanci", + "loni", "upřímný", "nečekal", "stoupne", "dívala", "kopce", "bere", "domy", "přicházet", + "očekává", "silný", "déšť", "teploty", "klesnou", "nulu", "řidiči", "měli", "silnicích", "zvlášť", "opatrní", "stůl", "pojďte", + "mnou", "bychom", "nejdřív", "jídelní", "lístek", "vynikající", }; // differences between Czech and Slovak languages / Slovak words / please keep the words aligned between these languages for better comparison private static readonly string[] AutoDetectWordsSlovak = { ".*[Ôô].*", ".*[ä].*", "[Bb]yť", "[Ss]om", "[Ss]i", "[Jj]a", "[Mm]ať", "[Áá]no", "[Nn]ie", "[Nn]ič", "[Dd]eň", "[Ll]en", "[Čč]o", "[Aa]ko", "[Aa]?[Ll]ebo", "[Pp]ri", "[Pp]re", "[Pp]red.*", "([Íí]sť|[Ii](dem|de|deme|dú))", - "[Mm]edzi", "[Ee]šte", "[Čč]lovek", "[Pp]odľa", "[Ďď]alš(í|ia|ie)" - }; + "[Mm]edzi", "[Ee]šte", "[Čč]lovek", "[Pp]odľa", "[Ďď]alš(í|ia|ie)", + "bolo", "bola", "bol", "boli", "sme", "ste", "sú", "nikto", "niečo", "niekto", "všetko", "všetci", "kým", "keď", "ak", "pretože", "kto", "kedy", "prečo", + "koľko", "ktorý", "ktorá", "ktoré", "teraz", "zajtra", "iba", "veľmi", "dobre", "zle", "ďakujem", "prepáč", "prepáčte", "pani", "ľudia", "dievča", "dieťa", + "deti", "roky", "milujem", "peniaze", "rýchlo", "pomaly", "majú", "mal", "môžem", "môžeš", "môže", "môžeme", "chcem", "viem", "vieš", "hovorím", "hovoríš", "hovoril", "hovorili", "povedz", "povedal", "povedala", "videl", "videli", "poď", "choď", "ideme", "urobiť", "urobil", "robíš", + "mesto", "nečakal", "rieka", "stúpne", "pozerala", "neskôr", "prichádzať", "celú", "pracoval", "dlho", "neklam", "pozri", "ľúto", "porozprávať", "povedať", + "šancu", "vlani", "odo", "mňa", "úprimný", "pokojné", "továreň", "nezavreli", "cesty", "vodou", "berie", "domy", "kopca", + "sa", "stôl", "dvoch", "samozrejme", "poďte", "najprv", "vidieť", "jedálny", "lístok", "vynikajúca", "severe", "očakáva", "dážď", "klesnú", "vodiči", + "cestách", "obzvlášť", "dobrý", "mohli", "jedlo", "práve", "aj", }; private static readonly string[] AutoDetectWordsSlovenian = { @@ -478,7 +947,14 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "ugrabila", "ubila", "tvoja", "tudi", "tečna", "stvari", "rusko", "povedala", "obraz", "nalogo", "mislim", "govoriš", "Seveda", "Razmišljam", "Potem", "Nič", "Nisem", "Mogoče", "žensko", "žalostna", "človek", "Čisto", "znaš", "zlomi", "zgodilo", "zdaj", "zajtrk", "utrujena","ustrelila", "srečna", "čarovnik", "zaživeti", - "Torej", "plasti","kanjon", "več", "prišlo", "presenečeni", "največje" + "Torej", "plasti","kanjon", "več", "prišlo", "presenečeni", "največje", + "bo", "bil", "niso", "sva", "boš", "bomo", "boste", "bodo", "ki", "če", "ker", "kjer", "kaj", "kdo", "kdaj", "kateri", "katera", "tisti", "tista", "tukaj", + "danes", "jutri", "včeraj", "vedno", "nikoli", "nihče", "nekdo", "vse", "vsi", "vsak", "zelo", "še", "prav", "slabo", "živjo", "adijo", "gospod", "gospa", + "ljudje", "moški", "ženska", "fant", "punca", "dekle", "otrok", "otroci", "noč", "zjutraj", "enkrat", "življenje", "ljubim", "hitro", "počasi", "skupaj", + "imajo", "imel", "imela", "morem", "hočem", "hočeš", "hoče", "hočemo", "nočem", "vem", "veš", "vemo", "rekel", "videl", "pridi", "pojdi", "gremo", + "narediti", "naredil", "delaš", "preden", "pričakoval", "narasla", "pozneje", "prihajati", "hiše", "vodo", "tovarna", "zaprla", "celo", "delal", "pozno", + "poslušaj", "žal", "morava", "pogovoriti", "reči", "priložnost", "lani", "odnaša", "mostu", + "pričakujemo", "močan", "dež", "ponoči", "padle", "ničlo", "vozniki", "naj", "posebej", "previdni", "dober", "mizo", "sledite", "jedilnik", "nocoj", "odlična", "najprej", "videli", }; private static readonly string[] AutoDetectWordsIcelandic = @@ -555,13 +1031,19 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "viņš", "viņa", "mēs", "jūs", "viņi", "viņas", "būt", "neesmu", "esmu", "varu", "nevaru", "gribu", "gaidi", "domāju", "zinu", "nezinu", - "cilvēks", "pasaulē", "dzīve", "laiks" + "cilvēks", "pasaulē", "dzīve", "laiks", + "bija", "pirms", "tika", "neviens", "ka", "tik", "ir", "nav", "būs", "būtu", "esi", "esam", "esat", "arī", "jau", "vēl", "tagad", "šeit", "tur", + "mani", "tev", "tevi", "viņam", "viņai", "mums", "viņiem", "šis", "šī", "tas", "tā", "tie", "tās", "visi", "viss", "visu", "kaut", "kāds", "kāda", + "neko", "nekas", "nekad", "vienmēr", "labi", "laba", "sveiki", "čau", "atvaino", "piedod", "vari", "varam", "varat", "jābūt", "vajag", + "gribi", "zini", "domā", "saki", "teica", "teicu", "runā", "redzu", "redzi", "redzēju", "nāc", "nāk", "ej", "iet", "darīt", "izdarīju", "vīrs", "vīrietis", + "sieviete", "puika", "meitene", "bērns", "cilvēki", "diena", "dienu", "nakts", "nakti", "rīts", "rītā", "laiku", "gads", "gadu", "gadiem", "reizi", "mājās", + "mājas", "mīlu", "miris", "notika", "nauda", "naudu", "ātri", "lēnām", "kopā", "viens", "pati", "žēl", "nē", "pilsēta", "fabrika", "slēgta", "upe", + "celsies", "strauji", "tilts", "ceļi", "ūdens", "vēlāk", "strādāju", "nemelo", "iespēja", "godīgs", "vairāk", }; private static readonly string[] AutoDetectWordsLithuanian = { - "tavęs", "veidai", "apie", "jums", "Veidai", "Kaip", "kaip", "reikia", "Šūdas", "frensis", "Ačiū", "vilsonai", "Palauk", "Veidas", "viskas", "Tikrai", "manęs", "Tačiau", "žmogau", "Flagai", "Prašau", "Džiune", "Nakties", - "šviesybe", "Supratau", "komanda", "reikia", "apie", "Kodėl", "mūsų", "Ačiū", "vyksta" + "ir", "yra", "nėra", "aš", "jis", "ji", "jūs", "jie", "jos", "mane", "mano", "tau", "tave", "tavo", "jam", "jai", "mums", "jums", "jiems", "čia", "dabar", "jau", "dar", "tik", "taip", "labai", "gerai", "gal", "galbūt", "nes", "jei", "jeigu", "kai", "kada", "kodėl", "kaip", "kas", "ką", "kur", "kuris", "kuri", "kiek", "visi", "viskas", "viską", "nieko", "niekas", "niekada", "visada", "kažkas", "kažką", "kiti", "kitas", "šis", "ši", "tas", "tai", "tie", "tos", "ačiū", "prašau", "atsiprašau", "labas", "sveiki", "tikrai", "tiesa", "žinau", "žinai", "nežinau", "manau", "galiu", "gali", "galime", "negaliu", "noriu", "nori", "norėjau", "reikia", "turiu", "turi", "turėjai", "turėjo", "esu", "esi", "esame", "buvo", "buvau", "buvai", "būti", "būsiu", "sakiau", "sakė", "sakyk", "pasakyti", "matau", "matai", "mačiau", "eik", "eina", "ateik", "daryti", "padariau", "vyras", "moteris", "berniukas", "mergaitė", "vaikas", "žmogus", "žmonės", "diena", "dieną", "naktis", "naktį", "rytas", "ryte", "laikas", "laiko", "metai", "metų", "kartą", "namo", "namuose", "gyvenimas", "myliu", "miręs", "atsitiko", "pinigai", "pinigų", "greitai", "lėtai", "kartu", "vienas", "viena", "gaila", "šūdas", "velnias", "apie", "tačiau", "mūsų", "jūsų", "tavęs", "manęs", "palauk", "supratau", "vyksta", "miestas", "ramus", "gamykla", "užsidarė", "upė", "pakils", "tiltas", "dingo", "keliai", "vandeniu", "žiūrėjo", "neša", "namus", "vėliau", "numatė", "buvai", "dirbau", "nemeluok", "šansą", "sąžiningas", "turėjai", }; private static readonly string[] AutoDetectWordsEstonian = @@ -572,7 +1054,17 @@ public static string AutoDetectGoogleLanguage(Encoding encoding) "olen", "oota", "tahan", "mees", "naine", "poiss", "tüdruk", "ma", "sa", "tema", "meie", "teie", "nema", "nad", "see", "seda", "siin", "seal", "praegu", "veel", "päris", - "tõesti", "muidugi", "vabandust", "hee", "kuule" + "tõesti", "muidugi", "vabandust", "hee", "kuule", + "oli", "olnud", "olid", "oled", "oleme", "olete", "ole", "oleks", "enne", "kui", "keegi", "kedagi", "nii", "kes", "kus", "kuhu", + "millal", "sest", "aga", "siis", "juba", "ikka", "nüüd", "sinna", "siia", "minu", "mulle", "sinu", "sulle", "teda", "talle", "meid", + "meile", "teid", "teile", "neid", "neile", "kõik", "kõike", "hea", "võib", "võin", "võid", "võime", "saab", "saan", "saad", "saame", "peab", + "pean", "pead", "peame", "tahad", "tahab", "tean", "tead", "teab", "arvan", "arvad", "mõtlen", "ütlesin", "ütles", "ütlesid", "räägi", "rääkima", "rääkisin", + "näe", "näen", "näed", "nägin", "tule", "tuleb", "läheb", "teha", "tegin", "tegi", "tehtud", "inimene", "inimesed", "päev", "päeva", "öö", + "öösel", "hommik", "hommikul", "aeg", "aega", "aasta", "aastat", "aastaid", "kord", "korda", "kodu", "koju", "kodus", "elu", "armastan", "surnud", "juhtus", + "juhtunud", "raha", "hetk", "kiiresti", "aeglaselt", "koos", "üksi", "kahju", "olgu", "jah", "ei", "linn", "tehas", "suleti", "oodanud", "jõgi", "tõuseb", + "sild", "teed", "vesi", "majad", "hiljem", "terve", "valeta", "võimalus", + "homme", "oodatakse", "põhja", "poole", "tugevat", "vihma", "temperatuur", "langeb", "alla", "nulli", "autojuhid", "peaksid", "teedel", "eriti", + "ettevaatlikud", "olema", "õhtul", "täna", "laud", "kahele", "tulge", "kaasa", "võiksime", "menüüd", "näha", "kala", "suurepärane", "õhtust", }; private static readonly string[] AutoDetectWordsHindi = @@ -613,6 +1105,8 @@ private static string AutoDetectGoogleLanguage(string text, int bestCount, out i var best = string.Empty; var bestScore = -1; var counts = CountWords(text); + var cyrillicChecked = false; + string cyrillicByLetters = null; void Consider(string languageCode, int score) { @@ -623,6 +1117,18 @@ void Consider(string languageCode, int score) } } + // "mk", "sr", "uk", "bg", "ru" or null, scanning the letters once no matter how many blocks ask. + string CyrillicByLetters() + { + if (!cyrillicChecked) + { + cyrillicByLetters = GetCyrillicLanguageByLetters(text, counts); + cyrillicChecked = true; + } + + return cyrillicByLetters; + } + var count = counts.Get(AutoDetectWordsEnglish); if (count > bestCount) { @@ -633,14 +1139,17 @@ void Consider(string languageCode, int score) } } + var danishOnlyCount = counts.Get(AutoDetectWordsDanishOnly); + var norwegianOnlyCount = counts.Get(AutoDetectWordsNorwegianOnly); count = counts.Get(AutoDetectWordsDanish); if (count > bestCount) { - var norwegianCount = counts.Get(AutoDetectWordsNorwegianOnly); var dutchCount = counts.Get(AutoDetectWordsDutch); var swedishCount = counts.Get(AutoDetectWordsSwedish); var icelandicCount = counts.Get(AutoDetectWordsIcelandic); - if (norwegianCount < 2 && dutchCount < count && swedishCount < count) + // Norwegian-only spellings outnumbering the Danish-only ones say Norwegian, unless + // there are too few of them to mean anything (one odd word in a long Danish file). + if ((norwegianOnlyCount < 2 || norwegianOnlyCount <= danishOnlyCount) && dutchCount < count && swedishCount < count) { if (icelandicCount > count * 1.5) { @@ -656,10 +1165,9 @@ void Consider(string languageCode, int score) count = counts.Get(AutoDetectWordsNorwegian); if (count > bestCount) { - var danishCount = counts.Get(AutoDetectWordsDanishOnly); var dutchCount = counts.Get(AutoDetectWordsDutch); var swedishCount = counts.Get(AutoDetectWordsSwedish); - if (danishCount < 2 && dutchCount < count && swedishCount < count) + if ((danishOnlyCount < 2 || danishOnlyCount <= norwegianOnlyCount) && dutchCount < count && swedishCount < count) { Consider("no", count); } @@ -729,15 +1237,20 @@ void Consider(string languageCode, int score) } count = counts.Get(AutoDetectWordsDutch); - if (count > bestCount) + var afrikaansCount = counts.Get(AutoDetectWordsAfrikaans); + if (count > bestCount || afrikaansCount > bestCount) { - Consider("nl", count); - } + // Afrikaans text hits the Dutch list too (het, van, een, ...); the spelling tells. + var afrikaansOrDutch = GetAfrikaansOrDutchByLetters(text); + if (count > bestCount) + { + Consider(afrikaansOrDutch ?? "nl", count); + } - count = counts.Get(AutoDetectWordsAfrikaans); - if (count > bestCount) - { - Consider("af", count); // Afrikaans + if (afrikaansCount > bestCount) + { + Consider(afrikaansOrDutch ?? "af", afrikaansCount); // Afrikaans + } } count = counts.Get(AutoDetectWordsPolish); @@ -766,7 +1279,12 @@ void Consider(string languageCode, int score) var bulgarianCount = counts.Get(AutoDetectWordsBulgarian); var ukrainianCount = counts.Get(AutoDetectWordsUkrainian); var macedonianCount = counts.Get(AutoDetectWordsMacedonian); - if (bulgarianCount > count) + var cyrillic = CyrillicByLetters(); + if (cyrillic != null) + { + Consider(cyrillic, count); + } + else if (bulgarianCount > count) { if (ukrainianCount > bulgarianCount && ukrainianCount > macedonianCount) { @@ -807,14 +1325,19 @@ void Consider(string languageCode, int score) count = counts.Get(AutoDetectWordsUkrainian); if (count > bestCount) { - Consider("uk", count); // Ukrainian + Consider(CyrillicByLetters() ?? "uk", count); // Ukrainian } count = counts.Get(AutoDetectWordsBulgarian); if (count > bestCount) { var macedonianCount = counts.Get(AutoDetectWordsMacedonian); - if (macedonianCount > count) + var cyrillic = CyrillicByLetters(); + if (cyrillic != null) + { + Consider(cyrillic, count); + } + else if (macedonianCount > count) { Consider("mk", count); } @@ -837,7 +1360,7 @@ void Consider(string languageCode, int score) var farsiCount = counts.Get(AutoDetectWordsFarsi); if (hebrewCount < count && farsiCount < count) { - Consider("ar", count); // Arabic + Consider(GetArabicScriptLanguageByLetters(text) ?? "ar", count); // Arabic } } @@ -850,7 +1373,7 @@ void Consider(string languageCode, int score) count = counts.Get(AutoDetectWordsFarsi); if (count > bestCount) { - Consider("fa", count); // Farsi (Persian) + Consider(GetArabicScriptLanguageByLetters(text) ?? "fa", count); // Farsi (Persian) } count = counts.Get(AutoDetectWordsCroatianAndSerbian); @@ -968,15 +1491,46 @@ void Consider(string languageCode, int score) int slovakWordsCount = counts.Get(AutoDetectWordsSlovak); if (czechWordsCount >= slovakWordsCount) { - Consider("cs", count); // Czech + Consider("cs", Math.Max(count, czechWordsCount)); // Czech } else { - Consider("sk", count); // Slovak + Consider("sk", Math.Max(count, slovakWordsCount)); // Slovak } } } + // Czech and Slovak on their own lists too (the shared list above may miss a plain + // line), but not on a single hit: "[Jj]a", "[Nn]e" and "[Dd]en" are everyday words + // in German, Dutch and the Scandinavian languages as well. + count = counts.Get(AutoDetectWordsCzech); + if (count > bestCount && count > 1) + { + var slovakWordsCount = counts.Get(AutoDetectWordsSlovak); + if (slovakWordsCount > count) + { + Consider("sk", slovakWordsCount); + } + else + { + Consider("cs", count); + } + } + + count = counts.Get(AutoDetectWordsSlovak); + if (count > bestCount && count > 1) + { + var czechWordsCount = counts.Get(AutoDetectWordsCzech); + if (czechWordsCount > count) + { + Consider("cs", czechWordsCount); + } + else + { + Consider("sk", count); + } + } + count = counts.Get(AutoDetectWordsSlovenian); if (count > bestCount) { @@ -998,15 +1552,20 @@ void Consider(string languageCode, int score) } count = counts.Get(AutoDetectWordsLatvian); - if (count > bestCount * 1.2) + var lithuanianWordCount = counts.Get(AutoDetectWordsLithuanian); + if (count > bestCount * 1.2 || lithuanianWordCount > bestCount) { - Consider("lv", count); - } + // kas, kur, kad, ir, jau, ... are both languages'; the letters are not. + var baltic = GetBalticLanguageByLetters(text); + if (count > bestCount * 1.2) + { + Consider(baltic ?? "lv", count); + } - count = counts.Get(AutoDetectWordsLithuanian); - if (count > bestCount) - { - Consider("lt", count); + if (lithuanianWordCount > bestCount) + { + Consider(baltic ?? "lt", lithuanianWordCount); + } } count = counts.Get(AutoDetectWordsHindi); @@ -1018,7 +1577,7 @@ void Consider(string languageCode, int score) count = counts.Get(AutoDetectWordsUrdu); if (count > bestCount) { - Consider("ur", count); + Consider(GetArabicScriptLanguageByLetters(text) ?? "ur", count); } count = counts.Get(AutoDetectWordsSinhalese); @@ -1030,7 +1589,7 @@ void Consider(string languageCode, int score) count = counts.Get(AutoDetectWordsMacedonian); if (count > bestCount) { - Consider("mk", count); + Consider(CyrillicByLetters() ?? "mk", count); } count = counts.Get(AutoDetectWordsIcelandic); @@ -1857,15 +2416,22 @@ public static Encoding DetectAnsiEncoding(byte[] buffer) { try { + var russianEncoding = Encoding.GetEncoding(1251); // Cyrillic + var textEnc1251 = russianEncoding.GetString(buffer); + var counts1251 = CountWords(textEnc1251); + var cyrillicCount = counts1251.Get(AutoDetectWordsRussianShort) + counts1251.Get(AutoDetectWordsSerbianCyrillic) + + counts1251.Get(AutoDetectWordsBulgarianShort) + counts1251.Get(AutoDetectWordsMacedonian) + + counts1251.Get(AutoDetectWordsUkrainian); + + // Cyrillic bytes read as 1253 are Greek letters too, and two-letter Greek words + // (να, το, με, ...) match such junk by chance - so Greek has to beat the Cyrillic count. var greekEncoding = Encoding.GetEncoding(1253); // Greek - if (GetCount(greekEncoding.GetString(buffer), AutoDetectWordsGreek) > 5) + var greekCount = GetCount(greekEncoding.GetString(buffer), AutoDetectWordsGreek); + if (greekCount > 5 && greekCount > cyrillicCount) { return greekEncoding; } - var russianEncoding = Encoding.GetEncoding(1251); // Cyrillic - var textEnc1251 = russianEncoding.GetString(buffer); - var counts1251 = CountWords(textEnc1251); if (counts1251.Get(AutoDetectWordsRussianShort) > 5) // Russian { return russianEncoding; @@ -2285,7 +2851,8 @@ private static bool IsUtf8(byte[] buffer, out bool couldBeUtf8) return true; } - private static readonly char[] RightToLeftLetters = string.Concat(AutoDetectWordsArabic.Concat(AutoDetectWordsHebrew).Concat(AutoDetectWordsFarsi).Concat(AutoDetectWordsUrdu)).Distinct().ToArray(); + // Letters only: the lists also hold phrases (a space), joiners and vowel marks. + private static readonly char[] RightToLeftLetters = string.Concat(AutoDetectWordsArabic.Concat(AutoDetectWordsHebrew).Concat(AutoDetectWordsFarsi).Concat(AutoDetectWordsUrdu)).Where(char.IsLetter).Distinct().ToArray(); #if NET8_0_OR_GREATER private static readonly SearchValues RightToLeftLetterSearchValues = SearchValues.Create(RightToLeftLetters); @@ -2417,6 +2984,13 @@ public static string GetEncodingViaLetter(string text) return null; } + // Letters a single Latin-script language owns settle it outright. + var latin = GetLatinLanguageByUniqueLetters(text); + if (latin != null) + { + return latin; + } + // Check for unique characters first var hasGermanUnique = ContainsAny(text, GermanUniqueSet); var hasSwedishUnique = ContainsAny(text, SwedishOnlySet); @@ -2520,24 +3094,32 @@ public static string GetEncodingViaLetter(string text) } } - if (topCount == 1) + var result = LanguageCodes[firstTop]; + for (var p = 0; topCount > 1 && p < PriorityOrder.Length; p++) { - return LanguageCodes[firstTop]; + // Priority order for tie-breaking + var i = Array.IndexOf(LanguageCodes, PriorityOrder[p]); + if (i >= 0 && languageScores[i] == maxScore) + { + result = PriorityOrder[p]; + break; + } } - // Priority order for tie-breaking - foreach (var lang in PriorityOrder) + if (result == "ru") { - for (var i = 0; i < languageScores.Length; i++) - { - if (languageScores[i] == maxScore && LanguageCodes[i] == lang) - { - return lang; - } - } + // The Cyrillic letter set is Russian's, so every Cyrillic text lands here; + // the other Cyrillic languages have letters of their own that say otherwise. + return GetCyrillicLanguageByLetters(text, null) ?? "ru"; + } + + if (result == "ar") + { + // Same for the Arabic script: Persian and Urdu have letters Arabic lacks. + return GetArabicScriptLanguageByLetters(text) ?? "ar"; } - return LanguageCodes[firstTop]; + return result; } public static bool IsLanguageWithoutPeriods(string language) diff --git a/src/libse/Common/NameListIndex.cs b/src/libse/Common/NameListIndex.cs new file mode 100644 index 00000000000..6c022836308 --- /dev/null +++ b/src/libse/Common/NameListIndex.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; + +namespace Nikse.SubtitleEdit.Core.Common +{ + /// + /// A name list prepared for . + /// The casing pass searched the line once for every name - thousands per line - although a + /// name can only be replaced where it is a whole word. A name without separator characters + /// can only match a whole token of the line, and a name like "Mr. Smith" only where its first + /// word is a token, so names are looked up by the tokens of the line here. Build it once per + /// name list. + /// + public sealed class NameListIndex + { + /// + /// Every character a name match may start after (StrippableText.IsNameStartBoundary) or + /// end before (StrippableText.NameEndChars). None of them changes under case folding. + /// + private const string Separators = " -\"'>[“,.!?:;)]<”\r\n"; + private static readonly char[] SeparatorChars = Separators.ToCharArray(); + + private readonly Dictionary> _tokenNames = new Dictionary>(StringComparer.OrdinalIgnoreCase); + private readonly int[] _alwaysSearch; + + public List Names { get; } + + public NameListIndex(IEnumerable names) + { + Names = new List(names); + var alwaysSearch = new List(); + for (var i = 0; i < Names.Count; i++) + { + // A match starts at a token start and the name's first word (up to its first + // separator) is followed by that same separator in the line, so the token there + // equals the first word - index the name by it. A name starting with a separator + // has no first word and is always searched. + var name = Names[i]; + var separatorIndex = string.IsNullOrEmpty(name) ? 0 : name.IndexOfAny(SeparatorChars); + if (separatorIndex == 0) + { + alwaysSearch.Add(i); + continue; + } + + var firstWord = separatorIndex < 0 ? name : name.Substring(0, separatorIndex); + if (!_tokenNames.TryGetValue(firstWord, out var positions)) + { + positions = new List(1); + _tokenNames.Add(firstWord, positions); + } + + positions.Add(i); + } + + _alwaysSearch = alwaysSearch.ToArray(); + } + + /// + /// Fills with the positions (ascending, so names are tried in + /// list order) of the names that can match in . + /// + internal void GetCandidates(string text, List candidates) + { + candidates.Clear(); + candidates.AddRange(_alwaysSearch); + var added = false; + var i = 0; + while (i < text.Length) + { + if (Separators.IndexOf(text[i]) >= 0) + { + i++; + continue; + } + + var end = i + 1; + while (end < text.Length && Separators.IndexOf(text[end]) < 0) + { + end++; + } + + if (_tokenNames.TryGetValue(text.Substring(i, end - i), out var positions)) + { + candidates.AddRange(positions); + added = true; + } + + i = end; + } + + if (!added) + { + return; + } + + candidates.Sort(); + var count = 1; + for (var k = 1; k < candidates.Count; k++) + { + if (candidates[k] != candidates[count - 1]) + { + candidates[count++] = candidates[k]; + } + } + + candidates.RemoveRange(count, candidates.Count - count); + } + } +} diff --git a/src/libse/Common/StrippableText.cs b/src/libse/Common/StrippableText.cs index c02b017e015..d854f0ab6c3 100644 --- a/src/libse/Common/StrippableText.cs +++ b/src/libse/Common/StrippableText.cs @@ -139,7 +139,10 @@ private static string GetAndInsertNextId(List replaceIds, List r return id; } - private void ReplaceNames1Remove(IEnumerable nameList, List replaceIds, List replaceNames, List originalNames) + [ThreadStatic] + private static List _nameCandidates; + + private void ReplaceNames1Remove(IEnumerable nameList, NameListIndex nameIndex, List replaceIds, List replaceNames, List originalNames) { if (Post.StartsWith('.')) { @@ -150,62 +153,81 @@ private void ReplaceNames1Remove(IEnumerable nameList, List repl string lower = StrippedText.ToLowerInvariant(); GetNameStartMask(lower, out var startMaskLow, out var startMaskHigh); int idName = 0; - foreach (string name in nameList) + if (nameIndex != null) { - // A name can only be replaced where it starts at a name start (see "startOk" - // below), so a name whose first letter is at none of them can be skipped without - // searching - the name list has thousands of entries, and this loop used to scan - // the line once for every one of them. - if (name.Length > 0 && name[0] < 128 && !MayStartName(name[0], startMaskLow, startMaskHigh)) + // Only the names that are a token of the line (or contain a separator) can match. + var candidates = _nameCandidates ?? (_nameCandidates = new List()); + nameIndex.GetCandidates(lower, candidates); + var names = nameIndex.Names; + foreach (var position in candidates) { - continue; + ReplaceName(names[position], ref lower, startMaskLow, startMaskHigh, ref idName, replaceIds, replaceNames, originalNames); } - - // "lower" is already lower case, so an ignore-case search finds the same - // positions as the lower-cased name did - without allocating one string per name. - int start = lower.IndexOf(name, StringComparison.OrdinalIgnoreCase); - while (start >= 0 && start < lower.Length) + } + else + { + foreach (string name in nameList) { - bool startOk = start == 0 || IsNameStartBoundary(lower[start - 1]); + ReplaceName(name, ref lower, startMaskLow, startMaskHigh, ref idName, replaceIds, replaceNames, originalNames); + } + } - if (startOk && string.CompareOrdinal(name, "Don") == 0 && lower.AsSpan(start).StartsWith("don't".AsSpan(), StringComparison.Ordinal)) - { - startOk = false; - } + if (StrippedText.EndsWith('.')) + { + Post = "." + Post; + StrippedText = StrippedText.TrimEnd('.'); + } + } - if (startOk) - { - int end = start + name.Length; - bool endOk = end <= lower.Length; - if (endOk) - { - endOk = end == lower.Length || NameEndChars.Contains(lower[end]); - } + private void ReplaceName(string name, ref string lower, ulong startMaskLow, ulong startMaskHigh, ref int idName, List replaceIds, List replaceNames, List originalNames) + { + // A name can only be replaced where it starts at a name start (see "startOk" + // below), so a name whose first letter is at none of them can be skipped without + // searching - the name list has thousands of entries, and this loop used to scan + // the line once for every one of them. + if (name.Length > 0 && name[0] < 128 && !MayStartName(name[0], startMaskLow, startMaskHigh)) + { + return; + } - if (endOk && StrippedText.Length >= start + name.Length) - { - string originalName = StrippedText.Substring(start, name.Length); - originalNames.Add(originalName); - StrippedText = StrippedText.Remove(start, name.Length); - StrippedText = StrippedText.Insert(start, GetAndInsertNextId(replaceIds, replaceNames, name, idName++)); - lower = StrippedText.ToLowerInvariant(); - } - } - if (start + 3 > lower.Length) + // "lower" is already lower case, so an ignore-case search finds the same + // positions as the lower-cased name did - without allocating one string per name. + int start = lower.IndexOf(name, StringComparison.OrdinalIgnoreCase); + while (start >= 0 && start < lower.Length) + { + bool startOk = start == 0 || IsNameStartBoundary(lower[start - 1]); + + if (startOk && string.CompareOrdinal(name, "Don") == 0 && lower.AsSpan(start).StartsWith("don't".AsSpan(), StringComparison.Ordinal)) + { + startOk = false; + } + + if (startOk) + { + int end = start + name.Length; + bool endOk = end <= lower.Length; + if (endOk) { - start = lower.Length + 1; + endOk = end == lower.Length || NameEndChars.Contains(lower[end]); } - else + + if (endOk && StrippedText.Length >= start + name.Length) { - start = lower.IndexOf(name, start + 3, StringComparison.OrdinalIgnoreCase); + string originalName = StrippedText.Substring(start, name.Length); + originalNames.Add(originalName); + StrippedText = StrippedText.Remove(start, name.Length); + StrippedText = StrippedText.Insert(start, GetAndInsertNextId(replaceIds, replaceNames, name, idName++)); + lower = StrippedText.ToLowerInvariant(); } } - } - - if (StrippedText.EndsWith('.')) - { - Post = "." + Post; - StrippedText = StrippedText.TrimEnd('.'); + if (start + 3 > lower.Length) + { + start = lower.Length + 1; + } + else + { + start = lower.IndexOf(name, start + 3, StringComparison.OrdinalIgnoreCase); + } } } @@ -306,11 +328,25 @@ private void ReplaceNames2Fix(List replaceIds, List replaceNames private static readonly char[] ExpectedCharsArray = { '.', '!', '?', ':', ';', ')', ']', '}', '(', '[', '{' }; public void FixCasing(IEnumerable nameList, bool changeNameCases, bool makeUppercaseAfterBreak, bool checkLastLine, string lastLine, double millisecondsFromLast = 0) + { + FixCasing(nameList, null, changeNameCases, makeUppercaseAfterBreak, checkLastLine, lastLine, millisecondsFromLast); + } + + /// + /// Same as the name list overload, but only searches the names that can match the line - + /// build the once and reuse it for every line. + /// + public void FixCasing(NameListIndex nameIndex, bool changeNameCases, bool makeUppercaseAfterBreak, bool checkLastLine, string lastLine, double millisecondsFromLast = 0) + { + FixCasing(null, nameIndex, changeNameCases, makeUppercaseAfterBreak, checkLastLine, lastLine, millisecondsFromLast); + } + + private void FixCasing(IEnumerable nameList, NameListIndex nameIndex, bool changeNameCases, bool makeUppercaseAfterBreak, bool checkLastLine, string lastLine, double millisecondsFromLast) { var replaceIds = new List(); var replaceNames = new List(); var originalNames = new List(); - ReplaceNames1Remove(nameList, replaceIds, replaceNames, originalNames); + ReplaceNames1Remove(nameList, nameIndex, replaceIds, replaceNames, originalNames); ReplaceAssaTagsRemove(replaceIds, replaceNames, originalNames); if (checkLastLine && ShouldStartWithUpperCase(lastLine, millisecondsFromLast)) diff --git a/src/libse/Common/Subtitle.cs b/src/libse/Common/Subtitle.cs index 5ce5edaf42b..549f998182b 100644 --- a/src/libse/Common/Subtitle.cs +++ b/src/libse/Common/Subtitle.cs @@ -305,7 +305,7 @@ public SubtitleFormat LoadSubtitle(string fileName, out Encoding encoding, Encod } } - if (useThisEncoding == null) + if (useThisEncoding == null && MayBeUtf16(fileName, encoding)) { return LoadSubtitle(fileName, out encoding, Encoding.Unicode); } @@ -313,6 +313,33 @@ public SubtitleFormat LoadSubtitle(string fileName, out Encoding encoding, Encod return null; } + /// + /// The UTF-16 retry runs every format check a second time, so only try it when it can + /// find something new: not when the file was already read as UTF-16, and not when its + /// start has no zero byte (UTF-16 Latin, digits and punctuation always have one). + /// + private static bool MayBeUtf16(string fileName, Encoding firstEncoding) + { + if (firstEncoding != null && firstEncoding.CodePage == Encoding.Unicode.CodePage) + { + return false; + } + + try + { + using (var fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + var buffer = new byte[4096]; + var count = fs.Read(buffer, 0, buffer.Length); + return Array.IndexOf(buffer, (byte)0, 0, count) >= 0; + } + } + catch + { + return true; + } + } + /// /// Asks one format whether the lines are its own. A damaged or truncated file can make a /// reader throw (an unclosed xml element, a json string with no end quote, ...), and this diff --git a/src/libse/Common/Utilities.cs b/src/libse/Common/Utilities.cs index c0b4c9e1398..8ae921bb8e6 100644 --- a/src/libse/Common/Utilities.cs +++ b/src/libse/Common/Utilities.cs @@ -1,5 +1,6 @@ using Nikse.SubtitleEdit.Core.Common.TextLengthCalculator; using Nikse.SubtitleEdit.Core.ContainerFormats.Matroska; +using Nikse.SubtitleEdit.Core.Dictionaries; using Nikse.SubtitleEdit.Core.SubtitleFormats; using Nikse.SubtitleEdit.Core.VobSub; using SkiaSharp; @@ -1374,19 +1375,9 @@ public static void RemoveFromUserDictionary(string word, string languageName) } } - words.Sort(); - - if (userWords.DocumentElement != null) - { - userWords.DocumentElement.RemoveAll(); - foreach (string w in words) - { - XmlNode node = userWords.CreateElement("word"); - node.InnerText = w; - userWords.DocumentElement.AppendChild(node); - } - } - + // Only the accepted words are rewritten - RemoveAll() also dropped the + // section (words the dictionary should reject). + UserDictionaryXml.WriteWords(userWords, words); userWords.Save(userWordsXmlFileName); } } @@ -1437,15 +1428,8 @@ public static void AddToUserDictionary(string word, string languageName) words.Add(word); } - words.Sort(); - - userWords.DocumentElement.RemoveAll(); - foreach (string w in words) - { - XmlNode node = userWords.CreateElement("word"); - node.InnerText = w; - userWords.DocumentElement.AppendChild(node); - } + UserDictionaryXml.WriteWords(userWords, words); + UserDictionaryXml.RemoveFromRemovedWords(userWords, word); } userWords.Save(userWordsXmlFileName); @@ -2332,12 +2316,6 @@ public static SKColor GetColorFromFontString(string text, SKColor defaultColor) } } - // Helper method to replace RemoveChar extension method - private static string Replace(this string input, char oldChar, string newString) - { - return input.Replace(oldChar.ToString(), newString); - } - public static SKColor FromHtml(string htmlColor) { return ColorTranslator.FromHtml(htmlColor); diff --git a/src/libse/ContainerFormats/Mp4/Boxes/Box.cs b/src/libse/ContainerFormats/Mp4/Boxes/Box.cs index 4659d214ab9..72c5ef522bc 100644 --- a/src/libse/ContainerFormats/Mp4/Boxes/Box.cs +++ b/src/libse/ContainerFormats/Mp4/Boxes/Box.cs @@ -67,6 +67,20 @@ public static string GetString(byte[] buffer, int index, int count) return text; } + /// + /// Reads a box payload whose length comes from the box size in the file. A corrupt size + /// asked for gigabytes (OutOfMemoryException), so cap it by the bytes left in the stream. + /// The few extra bytes keep the zero padding that the "Size - 4" reads of the last box in + /// a file have always relied on. + /// + internal static byte[] ReadPayload(System.IO.Stream fs, ulong length) + { + var bytesLeft = fs.Length > fs.Position ? (ulong)(fs.Length - fs.Position) : 0UL; + var buffer = new byte[Math.Min(length, bytesLeft + 8)]; + fs.ReadFully(buffer, 0, buffer.Length); + return buffer; + } + internal bool InitializeSizeAndName(System.IO.Stream fs) { if (StartPosition == 0) diff --git a/src/libse/ContainerFormats/Mp4/Boxes/Elst.cs b/src/libse/ContainerFormats/Mp4/Boxes/Elst.cs index abf1a0d8abb..336cfd144dd 100644 --- a/src/libse/ContainerFormats/Mp4/Boxes/Elst.cs +++ b/src/libse/ContainerFormats/Mp4/Boxes/Elst.cs @@ -33,8 +33,7 @@ public Elst(Stream fs, ulong size) return; } - Buffer = new byte[size - 8]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, size - 8); var version = Buffer[0]; var entryCount = GetUInt(4); diff --git a/src/libse/ContainerFormats/Mp4/Boxes/Mdia.cs b/src/libse/ContainerFormats/Mp4/Boxes/Mdia.cs index 69d8ed19d5e..34176f6b96f 100644 --- a/src/libse/ContainerFormats/Mp4/Boxes/Mdia.cs +++ b/src/libse/ContainerFormats/Mp4/Boxes/Mdia.cs @@ -56,8 +56,7 @@ public Mdia(Stream fs, ulong maximumLength) } else if (Name == "hdlr") { - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); HandlerType = GetString(8, 4); if (Size > 25) { diff --git a/src/libse/ContainerFormats/Mp4/Boxes/Stbl.cs b/src/libse/ContainerFormats/Mp4/Boxes/Stbl.cs index c83a8ee7c88..413b51dd46b 100644 --- a/src/libse/ContainerFormats/Mp4/Boxes/Stbl.cs +++ b/src/libse/ContainerFormats/Mp4/Boxes/Stbl.cs @@ -175,8 +175,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, { if (handlerType != "soun") { - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); int version = Buffer[0]; var totalEntries = GetUInt(4); @@ -196,8 +195,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, { if (handlerType != "soun") { - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); int version = Buffer[0]; var totalEntries = GetUInt(4); @@ -213,8 +211,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, } else if (Name == "stsz") // sample sizes { - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); int version = Buffer[0]; var uniformSizeOfEachSample = GetUInt(4); var numberOfSampleSizes = GetUInt(8); @@ -249,8 +246,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, // packed into 4, 8 or 16 bits each instead of a full word. Bento4's // mp4compact rewrites a file this way, and without this the track came out // with no sample sizes at all, i.e. no subtitles. - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); var fieldSize = Buffer[7]; // 3 bytes reserved, then the field size var sampleCount = GetUInt(8); StszSampleCount = sampleCount; @@ -283,8 +279,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, { //https://developer.apple.com/library/mac/#documentation/QuickTime/QTFF/QTFFChap2/qtff2.html#//apple_ref/doc/uid/TP40000939-CH204-SW1 - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); int version = Buffer[0]; var numberOfSampleTimes = GetUInt(4); var entries = ClampEntries(numberOfSampleTimes, 15, 8); @@ -310,8 +305,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, } else if (Name == "ctts") // composition time offset (PTS = DTS + offset); needed when B-frames make storage order differ from display order { - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); int version = Buffer[0]; var numberOfEntries = GetUInt(4); var entries = ClampEntries(numberOfEntries, 15, 8); @@ -334,8 +328,7 @@ public Stbl(Stream fs, ulong maximumLength, ulong timeScale, string handlerType, } else if (Name == "stsc") // sample table sample to chunk map { - Buffer = new byte[Size - 4]; - fs.ReadFully(Buffer, 0, Buffer.Length); + Buffer = ReadPayload(fs, Size - 4); int version = Buffer[0]; var numberOfSampleTimes = GetUInt(4); var entries = ClampEntries(numberOfSampleTimes, 20, 12); diff --git a/src/libse/ContainerFormats/Mp4/Boxes/Trun.cs b/src/libse/ContainerFormats/Mp4/Boxes/Trun.cs index f91712dd95b..e4bd10642b3 100644 --- a/src/libse/ContainerFormats/Mp4/Boxes/Trun.cs +++ b/src/libse/ContainerFormats/Mp4/Boxes/Trun.cs @@ -13,6 +13,9 @@ public class Trun : Box public List Samples { get; set; } public uint? DataOffset { get; set; } + // Far above a real fragment (a three-hour 60 fps video in a single fragment is ~650k samples). + private const ulong MaxSamplesWithoutPerSampleFields = 1_000_000; + public Trun(Stream fs, ulong maximumLength) { Samples = new List(); @@ -55,6 +58,16 @@ public Trun(Stream fs, ulong maximumLength) // Cap by the bytes actually left in the stream (never smaller than the real table, so // valid files are unaffected) and do the multiply in 64-bit so it cannot wrap. var bytesLeftInFile = fs.Length > fs.Position ? (ulong)(fs.Length - fs.Position) : 0UL; + + // Without per-sample fields (all values come from the tfhd/trex defaults) the loop below + // reads nothing, so only sample_count bounded it: a corrupt trun declaring 4 billion + // samples hung the parser while it filled the list. Every sample still needs at least + // one byte of media data, so it cannot outnumber the bytes left in the file. + if ((flags & (0x000100 | 0x000200 | 0x000400 | 0x000800)) == 0) + { + sampleCount = (uint)Math.Min(sampleCount, Math.Min(bytesLeftInFile, MaxSamplesWithoutPerSampleFields)); + } + var sampleLength = Math.Min((ulong)sampleCount * 16UL + 24UL, bytesLeftInFile); sampleLength = Math.Min(sampleLength, int.MaxValue); Buffer = new byte[(int)sampleLength]; diff --git a/src/libse/Dictionaries/UserDictionaryXml.cs b/src/libse/Dictionaries/UserDictionaryXml.cs new file mode 100644 index 00000000000..e2b32dabf8e --- /dev/null +++ b/src/libse/Dictionaries/UserDictionaryXml.cs @@ -0,0 +1,149 @@ +using Nikse.SubtitleEdit.Core.Common; +using System.Collections.Generic; +using System.Xml; + +namespace Nikse.SubtitleEdit.Core.Dictionaries +{ + /// + /// Reads and writes the two word lists of a "<lang>_user.xml" / "<lang>_se.xml" file: + /// the words the spell checker should accept, and the words it should reject even when the + /// Hunspell dictionary knows them (e.g. Dutch "weetje", which hides the OCR error "weet je"). + /// + /// <words> + /// <word>accepted</word> + /// <removed> + /// <word>weetje</word> + /// </removed> + /// </words> + /// + /// The removed words sit one level down, so older versions reading "word" children of the + /// root never see them as accepted words. + /// + public static class UserDictionaryXml + { + public const string RemovedElementName = "removed"; + + public static List ReadWords(XmlDocument doc) + { + return ReadNormalizedWords(doc.DocumentElement); + } + + public static List ReadRemovedWords(XmlDocument doc) + { + return ReadNormalizedWords(doc.DocumentElement?.SelectSingleNode(RemovedElementName)); + } + + /// + /// Replaces the accepted words and keeps everything else in the file (the removed words). + /// + public static void WriteWords(XmlDocument doc, List words) + { + var root = doc.DocumentElement; + if (root == null) + { + return; + } + + RemoveWordNodes(root); + words.Sort(); + var anchor = root.FirstChild; + foreach (var word in words) + { + XmlNode node = doc.CreateElement("word"); + node.InnerText = word; + if (anchor == null) + { + root.AppendChild(node); + } + else + { + root.InsertBefore(node, anchor); + } + } + } + + public static void WriteRemovedWords(XmlDocument doc, List words) + { + var root = doc.DocumentElement; + if (root == null) + { + return; + } + + var removedNode = root.SelectSingleNode(RemovedElementName); + if (removedNode == null) + { + if (words.Count == 0) + { + return; + } + + removedNode = doc.CreateElement(RemovedElementName); + root.AppendChild(removedNode); + } + + RemoveWordNodes(removedNode); + if (words.Count == 0) + { + root.RemoveChild(removedNode); + return; + } + + words.Sort(); + foreach (var word in words) + { + XmlNode node = doc.CreateElement("word"); + node.InnerText = word; + removedNode.AppendChild(node); + } + } + + /// + /// Adding a word to one list takes it out of the other, so a word is never both + /// accepted and rejected by the same file. + /// + public static void RemoveFromRemovedWords(XmlDocument doc, string normalizedWord) + { + var removed = ReadRemovedWords(doc); + if (removed.Remove(normalizedWord)) + { + WriteRemovedWords(doc, removed); + } + } + + private static List ReadNormalizedWords(XmlNode parent) + { + var list = new List(); + var nodes = parent?.SelectNodes("word"); + if (nodes == null) + { + return list; + } + + foreach (XmlNode node in nodes) + { + var word = Utilities.NormalizeUserDictionaryWord(node.InnerText); + if (word.Length > 0 && !list.Contains(word)) + { + list.Add(word); + } + } + + return list; + } + + private static void RemoveWordNodes(XmlNode parent) + { + var nodes = parent.SelectNodes("word"); + if (nodes == null) + { + return; + } + + foreach (XmlNode node in nodes) + { + parent.RemoveChild(node); + } + } + } +} diff --git a/src/libse/Settings/SubtitleSettings.cs b/src/libse/Settings/SubtitleSettings.cs index 833ef99d792..0d872f064fd 100644 --- a/src/libse/Settings/SubtitleSettings.cs +++ b/src/libse/Settings/SubtitleSettings.cs @@ -77,6 +77,22 @@ public class SubtitleSettings public string TimedTextItunesTimeCodeFormat { get; set; } public string TimedTextItunesStyleAttribute { get; set; } public string TimedTextItunesLanguage { get; set; } + + /// tts:color for the styles of the built-in iTunes Timed Text header (when no header is carried over). + public string TimedTextItunesDefaultColor { get; set; } + + /// Style id written on <body> (empty = none). With a template header it overrides the template's own. + public string TimedTextItunesDefaultStyle { get; set; } + + /// Region id written on <body> (empty = none). With a template header it overrides the template's own. + public string TimedTextItunesDefaultRegion { get; set; } + + /// + /// A complete TTML document whose head (styling, layout, metadata) and body attributes are + /// used for every saved iTunes Timed Text file instead of the source file's header. Empty = + /// keep the source header (or the built-in one). The UI loads it from the user's template file. + /// + public string TimedTextItunesHeaderTemplate { get; set; } public string TimedTextImsc11TimeCodeFormat { get; set; } public string TimedTextImsc11FileExtension { get; set; } @@ -166,6 +182,10 @@ public SubtitleSettings() TimedTextItunesBottomExtent = "100% 15%"; TimedTextItunesTimeCodeFormat = "Frames"; TimedTextItunesStyleAttribute = "tts:fontStyle"; + TimedTextItunesDefaultColor = "white"; + TimedTextItunesDefaultStyle = string.Empty; + TimedTextItunesDefaultRegion = string.Empty; + TimedTextItunesHeaderTemplate = string.Empty; TimedTextImsc11TimeCodeFormat = "hh:mm:ss.ms"; TimedTextImsc11FileExtension = ".xml"; diff --git a/src/libse/SubtitleFormats/CsvExcel.cs b/src/libse/SubtitleFormats/CsvExcel.cs index 9d945c921cf..b62239e97d0 100644 --- a/src/libse/SubtitleFormats/CsvExcel.cs +++ b/src/libse/SubtitleFormats/CsvExcel.cs @@ -15,7 +15,6 @@ namespace Nikse.SubtitleEdit.Core.SubtitleFormats public class CsvExcel : SubtitleFormat { private const char Separator = ','; - private const int NumberIndex = 0; private const int StartIndex = 1; private const int EndIndex = 2; private const int TextIndex = 3; diff --git a/src/libse/SubtitleFormats/ItunesTimedText.cs b/src/libse/SubtitleFormats/ItunesTimedText.cs index b6f3dbc4434..60ac3fa6705 100644 --- a/src/libse/SubtitleFormats/ItunesTimedText.cs +++ b/src/libse/SubtitleFormats/ItunesTimedText.cs @@ -1,4 +1,5 @@ using Nikse.SubtitleEdit.Core.Common; +using Nikse.SubtitleEdit.Core.Settings; using System; using System.Collections.Generic; using System.Linq; @@ -35,9 +36,17 @@ public override bool IsMine(List lines, string fileName) public override string ToText(Subtitle subtitle, string title) { + var settings = Configuration.Settings.SubtitleSettings; + + // A template header (File > Properties > "Header source: template file") replaces whatever + // header the source file carries, so every saved .itt gets the same styling/layout (#16257). + var templateXml = LoadHeaderTemplate(settings.TimedTextItunesHeaderTemplate); + var useTemplate = templateXml != null; + var sourceIsSubStationAlpha = subtitle.Header != null && (subtitle.Header.Contains("[V4+ Styles]") || subtitle.Header.Contains("[V4 Styles]")); + XmlNode styleHead = null; var convertedFromSubStationAlpha = false; - if (subtitle.Header != null) + if (!useTemplate && subtitle.Header != null) { try { @@ -91,11 +100,7 @@ public override string ToText(Subtitle subtitle, string title) } var xml = new XmlDocument { XmlResolver = null }; - var nsmgr = new XmlNamespaceManager(xml.NameTable); - nsmgr.AddNamespace("ttml", "http://www.w3.org/ns/ttml"); - nsmgr.AddNamespace("ttp", "http://www.w3.org/ns/10/ttml#parameter"); - nsmgr.AddNamespace("tts", "http://www.w3.org/ns/ttml#styling"); - nsmgr.AddNamespace("ttm", "http://www.w3.org/ns/10/ttml#metadata"); + var nsmgr = MakeNamespaceManager(xml); var frameRate = ((int)Math.Round(Configuration.Settings.General.CurrentFrameRate)).ToString(); var frameRateMultiplier = "999 1000"; @@ -110,30 +115,60 @@ public override string ToText(Subtitle subtitle, string title) } var language = LanguageAutoDetect.AutoDetectGoogleLanguage(subtitle); - if (!string.IsNullOrEmpty(Configuration.Settings.SubtitleSettings.TimedTextItunesLanguage)) + if (!string.IsNullOrEmpty(settings.TimedTextItunesLanguage)) + { + language = settings.TimedTextItunesLanguage; + } + + var defaultColor = XmlAttributeValueOrDefault(settings.TimedTextItunesDefaultColor, "white"); + var bodyAttributes = string.Empty; + if (BuiltInStyles.Contains(settings.TimedTextItunesDefaultStyle ?? string.Empty)) + { + bodyAttributes += " style=\"" + settings.TimedTextItunesDefaultStyle + "\""; + } + + if (BuiltInRegions.Contains(settings.TimedTextItunesDefaultRegion ?? string.Empty)) { - language = Configuration.Settings.SubtitleSettings.TimedTextItunesLanguage; + bodyAttributes += " region=\"" + settings.TimedTextItunesDefaultRegion + "\""; } var xmlStructure = "" + Environment.NewLine + "" + Environment.NewLine + " " + Environment.NewLine + " " + Environment.NewLine + - "