From d7a30a74c4f06c204b03223f7f64fa8973354120 Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 00:46:44 +0700 Subject: [PATCH 01/10] test: validate book chapter headings, timelines and stubs --- scripts/check_chapter_structure.py | 75 ++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scripts/check_chapter_structure.py diff --git a/scripts/check_chapter_structure.py b/scripts/check_chapter_structure.py new file mode 100644 index 0000000..bb89f76 --- /dev/null +++ b/scripts/check_chapter_structure.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Validate Book 01 technical-literature chapter contracts (stdlib only).""" +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import sys + +ENDING = ( + "Rules worth keeping in working memory", + "Diagnostics, useful compiler settings and extensions", + "Standards timeline", + "Migration note for Java / Python / C# developers", + "Further reading", +) +YEARS = ("C++98/03", "C++11", "C++14", "C++17", "C++20", "C++23", "C++26") +STUBS = re.compile(r"\b(?:TODO|TBD|FIXME|PLACEHOLDER|fill this in)\b|", re.I) +HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.M) +FENCE = re.compile(r"^\s*(`{3,}|~{3,})", re.M) +CHAPTER = re.compile(r"^(\d{2})_") + +def validate(path: Path) -> list[str]: + body = path.read_text(encoding="utf-8") + errors: list[str] = [] + chapter = CHAPTER.match(path.parent.name) + if not chapter: + return [f"{path}: not a numbered chapter directory"] + headings = [(len(m.group(1)), m.group(2)) for m in HEADING.finditer(body)] + if not headings or headings[0][0] != 1 or not headings[0][1].startswith(chapter.group(1)): + errors.append("H1 must start with chapter number") + h2 = [h for level, h in headings if level == 2] + if h2[-len(ENDING):] != list(ENDING): + errors.append("final five H2 sections must match the chapter template in order") + for h in ENDING: + if h2.count(h) != 1: + errors.append(f"expected exactly one H2: {h}") + numeric = [int(m.group(1)) for h in h2 if (m := re.match(r"^(\d+)\.\s", h))] + if not numeric or numeric != list(range(1, len(numeric) + 1)): + errors.append(f"numbered H2 sections not consecutive: {numeric}") + found_stubs = STUBS.findall(body) + if found_stubs: + errors.append(f"contains stub markers: {found_stubs[:3]}") + fences = FENCE.findall(body) + if len(fences) % 2: + errors.append("unbalanced fenced code blocks") + timeline_start = body.find("\n## Standards timeline") + migration_start = body.find("\n## Migration note") + if timeline_start >= 0 and migration_start > timeline_start: + timeline = body[timeline_start:migration_start] + missing = [std for std in YEARS if not re.search(r"^### " + re.escape(std) + r"\b", timeline, re.M)] + if missing: + errors.append(f"standards timeline missing: {', '.join(missing)}") + if len(body) < 7000: + errors.append("chapter is unexpectedly short for advanced technical literature") + for item in errors: + print(f"{path}: {item}", file=sys.stderr) + if not errors: + print(f"PASS {path} ({len(numeric)} numbered sections)") + return errors + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("paths", nargs="*", type=Path, + help="README.md paths; defaults to chapters 01–13") + args = ap.parse_args() + paths = args.paths or sorted(Path("01_low_level").glob("[0-9][0-9]_*/README.md")) + if not paths: + print("No chapter READMEs found", file=sys.stderr) + return 2 + failures = sum(bool(validate(p)) for p in paths) + return 1 if failures else 0 + +if __name__ == "__main__": + raise SystemExit(main()) From 94d3e20bd7afe5c32e549f7cbfb66edf2cb0ff7c Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 00:46:46 +0700 Subject: [PATCH 02/10] ci: run chapter Markdown structure validator --- .github/workflows/chapter-structure.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/chapter-structure.yml diff --git a/.github/workflows/chapter-structure.yml b/.github/workflows/chapter-structure.yml new file mode 100644 index 0000000..2a08f62 --- /dev/null +++ b/.github/workflows/chapter-structure.yml @@ -0,0 +1,24 @@ +name: Chapter Markdown Structure +on: + pull_request: + paths: + - '01_low_level/**/README.md' + - 'scripts/check_chapter_structure.py' + - '.github/workflows/chapter-structure.yml' + push: + branches: [master] + paths: + - '01_low_level/**/README.md' + - 'scripts/check_chapter_structure.py' + - '.github/workflows/chapter-structure.yml' + workflow_dispatch: +jobs: + chapter-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Validate Book 01 chapter structure + run: python scripts/check_chapter_structure.py From c9fbeb6ec36109866ed19601bf7a0fd5be501b3b Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 00:48:47 +0700 Subject: [PATCH 03/10] docs: extend bitwise chapter through C++26 and deepen senior pitfalls --- 01_low_level/08_bitwise_ops/README.md | 144 +++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 4 deletions(-) diff --git a/01_low_level/08_bitwise_ops/README.md b/01_low_level/08_bitwise_ops/README.md index 70dd364..9f134e4 100644 --- a/01_low_level/08_bitwise_ops/README.md +++ b/01_low_level/08_bitwise_ops/README.md @@ -1,6 +1,8 @@ -# 08. Bitwise Operations in C++ +# 08. Bitwise Operations from C++98 to C++26 -## Advanced Facts, Pitfalls, and Evolution (C++98 → C++20) +Bitwise expressions operate on integral values, but their safety hinges on promotions, signedness, shift counts, and object representation. This chapter develops the C++98 language rules before the modern standard vocabulary, and separates value-level bit patterns from byte layouts and hardware register protocols. + +The seven example programs cover representation, shifts, arithmetic tricks, bit scans, flags, standard containers, and radix/layout work. New C++23/26 material is feature-dependent; the repository continues to build as C++20. > Bitwise operations are where C++ stops abstracting and starts describing the machine. > This chapter documents *what is guaranteed*, *what is merely common*, and *what is undefined*. @@ -333,6 +335,136 @@ enum class Perm : uint32_t { Read = 1, Write = 2 }; --- --- +## 15. Rotate, endian conversion, and bit fields are three different problems + +A fixed-width integer can describe a mathematical bit sequence, a machine register, or serialized bytes. Those domains should not be conflated. The C++20 `` header contains bit-pattern operations, but converting to a wire representation still requires an explicit byte-order contract. + +### Rotation must not be implemented using undefined shifts + +The expression `(x << n) | (x >> (width - n))` appears everywhere in legacy cryptographic and hash functions. As written, it can shift by the exact type width when `n == 0`, which is undefined behavior. C++20's `std::rotl` and `std::rotr` give the intended rotation for unsigned integer types and normalize the rotation count. + +```cpp +#include +#include + +constexpr auto rotated = std::rotl(std::uint32_t{0x12345678u}, 8); +static_assert(rotated == 0x34567812u); +``` + +Use rotations for permutation and mixing, not as substitutes for arithmetic multiplication or for byte swapping. `std::rotl` is defined on unsigned integer types; when working with signed state, convert intentionally. + +### Bit position is not byte order + +A network packet specifies an order of octets, while a machine integer has a value and an object representation. Shifting an unsigned value extracts its mathematical bits regardless of host endian layout; copying its in-memory bytes does not. + +```cpp +constexpr std::uint32_t header = 0x12345678u; +constexpr auto first_wire_octet = (header >> 24) & 0xffu; +static_assert(first_wire_octet == 0x12u); +``` + +C++20 `std::endian` describes native scalar endianness, including the possibility of mixed endianness. C++23 `std::byteswap` reverses the bytes of an integer value; it does not automatically define a complete serialization policy for structs containing padding. + +### C++ bit-fields are not portable wire-format layouts + +Bit-field members can be useful for storage-conscious in-process structures, but their allocation order, packing, padding, and alignment are implementation-dependent. Even the address of a bit-field cannot be taken, and operations may involve integer promotions. + +```cpp +struct DeviceFlags { + unsigned ready : 1; + unsigned mode : 3; +}; +// Do not reinterpret this struct as a portable network header. +``` + +For external formats, decode explicit byte arrays with shifts and masks. For registers, follow the hardware vendor's documented access rules, including volatile-qualified MMIO where required; `volatile` alone is not thread synchronization. + +--- + +## 16. Bit scans have edge cases that change control flow + +Counting leading and trailing zeros is foundational for allocators, radix trees, compressed indexes, and integer codecs. The instruction-level implementations are often extremely fast, but historical compiler intrinsics disagree about the all-zero input. + +### Standard count functions define the zero case + +C++20 `std::countl_zero` and `std::countr_zero` return the width of the unsigned type for zero, while many pre-standard compiler builtins such as `__builtin_ctz(0)` have undefined behavior. + +```cpp +#include +#include +#include + +static_assert(std::countr_zero(std::uint32_t{0}) == 32); +static_assert(std::countl_zero(std::uint32_t{0}) == 32); +static_assert(std::bit_width(std::uint32_t{0}) == 0); +``` + +`std::bit_width` yields the number of bits needed to represent the value; it is *not* the number of leading zeros. Encoding a data structure's sentinel convention should happen above these primitives. + +### Population count applies to bits, not a signed mathematical value + +C++20 `std::popcount` accepts unsigned integer types. The result type is `int`, regardless of the unsigned operand width. Explicitly choose the intended representation when the original source is signed. + +```cpp +constexpr auto count = std::popcount(std::uint32_t{0b101101u}); +static_assert(count == 4); +``` + +Avoid the assumption that a negative value has a portable number of one-bits without specifying its unsigned conversion and width. A signed-to-unsigned conversion is well-defined modulo \(2^N\), but its bit-pattern interpretation depends on the chosen unsigned type. + +--- + +## 17. Power-of-two arithmetic has overflow boundaries + +A common pre-C++20 test is `x != 0 && (x & (x - 1)) == 0`. It is correct for unsigned integers and commonly used in page allocators and ring buffers. The more difficult problem is rounding a value *up* to a power of two without wrapping. + +### C++20 `has_single_bit` and `bit_ceil` expose intent + +`std::has_single_bit(x)` expresses the test directly. `std::bit_ceil(x)` returns the least power of two not less than `x`, but if that power cannot be represented in the result type its behavior is undefined. + +```cpp +#include +#include + +static_assert(std::has_single_bit(std::uint32_t{64})); +static_assert(std::bit_floor(std::uint32_t{70}) == 64); +static_assert(std::bit_ceil(std::uint32_t{70}) == 128); +``` + +Validate the maximum input before rounding allocation sizes. Do not replace one historical overflow-prone bit hack with a standard API while keeping the same unchecked precondition. + +### Bit-mask extraction is not necessarily an authorization check + +When a bitmask controls permissions, checking whether *any* requested bit is set is different from requiring *all* bits. A correct mask operation can still implement a security bug if the logical predicate is wrong. + +```cpp +constexpr bool all_bits(std::uint32_t got, std::uint32_t required) { + return (got & required) == required; +} +``` + +The empty requested mask returns true. Decide explicitly whether that matches the protocol's domain semantics. + +--- + +## 18. C++26 bit-oriented evolution and performance engineering + +C++26 provides additional data-parallel tools but does not change the classic meaning of unsigned shifts and masks. Engineering decisions still depend on representation, aliasing, memory bandwidth, and algorithm invariants. + +### `` can replace some manual lane-packing tricks + +C++26 standard data-parallel facilities provide explicit vector lanes and masks. Their usefulness is greatest when the algorithm processes independent elements and memory access satisfies alignment and lifetime preconditions. It is not correct to assume that an arbitrary packed struct can be reinterpreted as SIMD vectors. + +Compilers have long recognized many unsigned bit idioms and lowered them to rotate, population count, and bit-scan instructions. Compare `std::popcount`, `std::rotl`, and standard masks with handwritten patterns in optimized builds before claiming that a manually branchless expression is faster. + +### `std::bitset` and dynamic bit collections have different constraints + +A `std::bitset` has compile-time capacity and provides named bit operations; `std::vector` stores packed bits through proxy references and is not a contiguous range of ordinary `bool` objects. Neither automatically provides atomic per-bit operations, and concurrent mutation requires an explicit synchronization design. + +For lock-free bitmaps, use suitable `std::atomic` word operations and specify how writers claim and release individual bits. Check whole-word atomicity and the memory order needed for the state protected by each bit. + +--- + ## Rules worth keeping in working memory The central distinction in bitwise operations and shifts is between what the C++ type system guarantees, what the implementation commonly does, and what an API merely assumes. Carrying that distinction from declarations through operations avoids most of the surprises discussed above. @@ -490,9 +622,9 @@ No material change to core bitwise operators. `std::byteswap` adds portable byte-order reversal. -### C++26: Emerging improvements +### C++26: Data-parallel facilities and unchanged scalar semantics -Further bit facilities may appear, but signed overflow and invalid shifts still require care. +C++26 adds standard data-parallel types in ``, useful for explicitly vectorized lane processing and masks. The scalar bitwise semantics, zero-input scan rules, and unsigned-only `` preconditions remain unchanged. Feature-test library availability; older SIMD intrinsics remain vendor-specific. When documenting a facility, state its required language/library version rather than inferring support from the compiler's branding. @@ -534,6 +666,10 @@ Use these choices because they express the program's requirements, not merely be ## Further reading +* [C++20 bit utilities](https://en.cppreference.com/w/cpp/header/bit) +* [C++23 `std::byteswap`](https://en.cppreference.com/w/cpp/numeric/byteswap) +* [C++26 SIMD library](https://en.cppreference.com/w/cpp/numeric/simd) + These references document the underlying language rules and library contracts. They are starting points for checking precise preconditions; the chapter's examples explain how the rules interact. ### Standard and language reference From aa78eb89ab432a5e985c0bd905992886e2b8785d Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 00:50:08 +0700 Subject: [PATCH 04/10] ci: gate changed chapter READMEs on pull requests --- .github/workflows/chapter-structure.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/chapter-structure.yml b/.github/workflows/chapter-structure.yml index 2a08f62..f0b0042 100644 --- a/.github/workflows/chapter-structure.yml +++ b/.github/workflows/chapter-structure.yml @@ -17,8 +17,21 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Validate Book 01 chapter structure - run: python scripts/check_chapter_structure.py + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + mapfile -t files < <(git diff --name-only --diff-filter=ACMR "${{ github.event.pull_request.base.sha }}" HEAD -- '01_low_level/*/README.md') + if (( ${#files[@]} )); then + python scripts/check_chapter_structure.py "${files[@]}" + else + echo "No changed chapter READMEs to validate" + fi + else + python scripts/check_chapter_structure.py + fi From 36d20aeb75ed0372853a4163f668601f6d64d325 Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 01:01:28 +0700 Subject: [PATCH 05/10] fix: align historical bitwise examples with C++20/23 rules --- .../01_bits_representation/bits_representation.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/01_low_level/08_bitwise_ops/01_bits_representation/bits_representation.cpp b/01_low_level/08_bitwise_ops/01_bits_representation/bits_representation.cpp index 806688e..d8fb7e6 100644 --- a/01_low_level/08_bitwise_ops/01_bits_representation/bits_representation.cpp +++ b/01_low_level/08_bitwise_ops/01_bits_representation/bits_representation.cpp @@ -74,7 +74,9 @@ static void signed_vs_unsigned_model() std::cout << "s = " << s << " bits(s as unsigned) = " << bits_u(u) << "\n"; // In general: converting signed -> unsigned is well-defined (mod 2^N). - // Converting unsigned -> signed is implementation-defined if out of range. + // Since C++20, out-of-range unsigned -> signed conversion uses the unique + // value congruent modulo 2^N for the destination width. Earlier standards + // left out-of-range results implementation-defined. } static void twos_complement_identities_carefully() From 7352ad3b64c94dab48719ac41a2f14e6f4487097 Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 01:01:31 +0700 Subject: [PATCH 06/10] fix: align historical bitwise examples with C++20/23 rules --- 01_low_level/08_bitwise_ops/02_shifts/shifts.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/01_low_level/08_bitwise_ops/02_shifts/shifts.cpp b/01_low_level/08_bitwise_ops/02_shifts/shifts.cpp index f7021b7..caefb51 100644 --- a/01_low_level/08_bitwise_ops/02_shifts/shifts.cpp +++ b/01_low_level/08_bitwise_ops/02_shifts/shifts.cpp @@ -74,11 +74,13 @@ static void demonstrate_shift_rules() // 1) shifting by >= width is UB for built-in shifts // u = u << 32; // UB for uint32_t - // 2) left shift on signed that overflows is UB + // 2) signed left shifts have version-sensitive special rules, including + // older allowance for reaching the sign bit in selected cases. + // For portable bit manipulation, convert to unsigned before shifting. // int s = 1; - // s = s << 31; // UB (overflow / sign bit) + // s = s << 31; // do not use as a portable bit-mask idiom - // 3) right shift of negative signed is implementation-defined + // 3) right shift of negative signed is implementation-defined before C++20 // int neg = -8; // std::cout << (neg >> 1) << "\n"; // arithmetic shift on most compilers, not guaranteed pre-C++20 } @@ -90,7 +92,7 @@ static void demonstrate_signed_right_shift_is_not_portable() std::cout << "neg=" << neg << " bits(as unsigned)=" << bits_u(static_cast(neg)) << "\n"; int r = (neg >> 1); std::cout << "neg>>1=" << r << " bits(as unsigned)=" << bits_u(static_cast(r)) << "\n"; - std::cout << "(Most targets do arithmetic shift, but do not rely on it for portability.)\n"; + std::cout << "(C++20 specifies negative signed right shift rounding toward negative infinity; older standards leave it implementation-defined.)\n"; } static void safe_shift_helpers_demo() From 200d0e9e4c3be3ef3c77f5be60ed16bc93581109 Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 01:01:33 +0700 Subject: [PATCH 07/10] fix: align historical bitwise examples with C++20/23 rules --- .../08_bitwise_ops/03_arithmetic_tricks/arithmetic_tricks.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/01_low_level/08_bitwise_ops/03_arithmetic_tricks/arithmetic_tricks.cpp b/01_low_level/08_bitwise_ops/03_arithmetic_tricks/arithmetic_tricks.cpp index 4e4161d..ca27627 100644 --- a/01_low_level/08_bitwise_ops/03_arithmetic_tricks/arithmetic_tricks.cpp +++ b/01_low_level/08_bitwise_ops/03_arithmetic_tricks/arithmetic_tricks.cpp @@ -20,6 +20,7 @@ Build (C++17): #include #include #include +#include #include template @@ -87,6 +88,8 @@ std::pair add_u_with_overflow(U a, U b) int abs_portable(int x) { + if (x == std::numeric_limits::min()) + throw std::overflow_error("abs(INT_MIN) is not representable"); return (x < 0) ? -x : x; } From e606b20db53f494396e1b9af88fb80ea154c5325 Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 01:01:36 +0700 Subject: [PATCH 08/10] fix: align historical bitwise examples with C++20/23 rules --- .../04_popcount_and_scans/popcount_and_scans.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/01_low_level/08_bitwise_ops/04_popcount_and_scans/popcount_and_scans.cpp b/01_low_level/08_bitwise_ops/04_popcount_and_scans/popcount_and_scans.cpp index 5fd9936..a67fac0 100644 --- a/01_low_level/08_bitwise_ops/04_popcount_and_scans/popcount_and_scans.cpp +++ b/01_low_level/08_bitwise_ops/04_popcount_and_scans/popcount_and_scans.cpp @@ -160,6 +160,11 @@ static void demo_scans() std::cout << "countl_zero = " << std::countl_zero(x) << "\n"; std::cout << "rotl(x,5) = 0x" << std::hex << std::rotl(x, 5) << std::dec << "\n"; std::cout << "rotr(x,5) = 0x" << std::hex << std::rotr(x, 5) << std::dec << "\n"; +#if defined(__cpp_lib_byteswap) && __cpp_lib_byteswap >= 202110L + std::cout << "C++23 byteswap = 0x" << std::hex + << std::byteswap(std::uint32_t{0x12345678u}) + << std::dec << "\n"; +#endif } #endif #endif From 3c2c92cdf491f47d9378ba454706e1a89c1bb222 Mon Sep 17 00:00:00 2001 From: Yurii Cherkasov Date: Mon, 19 Oct 2026 01:03:42 +0700 Subject: [PATCH 09/10] ci: validate only changed chapters on push and PR; keep manual full-book audit --- .github/workflows/chapter-structure.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/chapter-structure.yml b/.github/workflows/chapter-structure.yml index f0b0042..f2afa8e 100644 --- a/.github/workflows/chapter-structure.yml +++ b/.github/workflows/chapter-structure.yml @@ -26,12 +26,16 @@ jobs: shell: bash run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then - mapfile -t files < <(git diff --name-only --diff-filter=ACMR "${{ github.event.pull_request.base.sha }}" HEAD -- '01_low_level/*/README.md') - if (( ${#files[@]} )); then - python scripts/check_chapter_structure.py "${files[@]}" - else - echo "No changed chapter READMEs to validate" - fi + base="${{ github.event.pull_request.base.sha }}" + elif [[ "${{ github.event_name }}" == "push" ]]; then + base="${{ github.event.before }}" else python scripts/check_chapter_structure.py + exit + fi + mapfile -t files < <(git diff --name-only --diff-filter=ACMR "$base" HEAD -- '01_low_level/*/README.md') + if (( ${#files[@]} )); then + python scripts/check_chapter_structure.py "${files[@]}" + else + echo "No changed chapter READMEs to validate" fi From e361ef249205dc6b445bd7beba1b88abaf580770 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:05:31 +0000 Subject: [PATCH 10/10] fix: parse chapter Markdown fences correctly Co-authored-by: yuchdev <678436+yuchdev@users.noreply.github.com> --- scripts/check_chapter_structure.py | 39 ++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/scripts/check_chapter_structure.py b/scripts/check_chapter_structure.py index bb89f76..cf90f8a 100644 --- a/scripts/check_chapter_structure.py +++ b/scripts/check_chapter_structure.py @@ -15,18 +15,42 @@ "Further reading", ) YEARS = ("C++98/03", "C++11", "C++14", "C++17", "C++20", "C++23", "C++26") -STUBS = re.compile(r"\b(?:TODO|TBD|FIXME|PLACEHOLDER|fill this in)\b|", re.I) +STUBS = re.compile(r"\b(?:TODO|TBD|FIXME|fill this in)\b|", re.I) HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.M) -FENCE = re.compile(r"^\s*(`{3,}|~{3,})", re.M) +FENCE_OPEN = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") +FENCE_CLOSE = re.compile(r"^ {0,3}(`{3,}|~{3,})[ \t]*$") CHAPTER = re.compile(r"^(\d{2})_") + +def outside_fences(body: str) -> tuple[str, bool]: + visible: list[str] = [] + opening: tuple[str, int] | None = None + for line in body.splitlines(keepends=True): + if opening is None: + match = FENCE_OPEN.match(line) + if match and not (match.group(1).startswith("`") and "`" in match.group(2)): + delimiter = match.group(1) + opening = (delimiter[0], len(delimiter)) + visible.append("\n" if line.endswith("\n") else "") + else: + visible.append(line) + continue + match = FENCE_CLOSE.match(line) + if (match and match.group(1)[0] == opening[0] + and len(match.group(1)) >= opening[1]): + opening = None + visible.append("\n" if line.endswith("\n") else "") + return "".join(visible), opening is not None + + def validate(path: Path) -> list[str]: body = path.read_text(encoding="utf-8") + visible_body, unbalanced_fences = outside_fences(body) errors: list[str] = [] chapter = CHAPTER.match(path.parent.name) if not chapter: return [f"{path}: not a numbered chapter directory"] - headings = [(len(m.group(1)), m.group(2)) for m in HEADING.finditer(body)] + headings = [(len(m.group(1)), m.group(2)) for m in HEADING.finditer(visible_body)] if not headings or headings[0][0] != 1 or not headings[0][1].startswith(chapter.group(1)): errors.append("H1 must start with chapter number") h2 = [h for level, h in headings if level == 2] @@ -41,13 +65,12 @@ def validate(path: Path) -> list[str]: found_stubs = STUBS.findall(body) if found_stubs: errors.append(f"contains stub markers: {found_stubs[:3]}") - fences = FENCE.findall(body) - if len(fences) % 2: + if unbalanced_fences: errors.append("unbalanced fenced code blocks") - timeline_start = body.find("\n## Standards timeline") - migration_start = body.find("\n## Migration note") + timeline_start = visible_body.find("\n## Standards timeline") + migration_start = visible_body.find("\n## Migration note") if timeline_start >= 0 and migration_start > timeline_start: - timeline = body[timeline_start:migration_start] + timeline = visible_body[timeline_start:migration_start] missing = [std for std in YEARS if not re.search(r"^### " + re.escape(std) + r"\b", timeline, re.M)] if missing: errors.append(f"standards timeline missing: {', '.join(missing)}")