From c7de7e1626a73d844a87df0ba1477528b618b451 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 13:35:14 +0000 Subject: [PATCH 01/46] Backport #124255 to 26.3: Bump vendored `expat` from 2.8.2 to 2.9.0 --- base/poco/XML/CMakeLists.txt | 2 + base/poco/XML/include/Poco/XML/expat.h | 105 +- .../XML/include/Poco/XML/expat_external.h | 17 + base/poco/XML/src/ascii.h | 2 + base/poco/XML/src/asciitab.h | 2 + base/poco/XML/src/fallthrough.h | 2 + base/poco/XML/src/hash_table.h | 76 ++ base/poco/XML/src/iasciitab.h | 2 + base/poco/XML/src/internal.h | 157 +-- base/poco/XML/src/latin1tab.h | 2 + base/poco/XML/src/nametab.h | 2 + base/poco/XML/src/siphash.h | 2 + base/poco/XML/src/utf8tab.h | 2 + base/poco/XML/src/xcs.h | 52 + base/poco/XML/src/xmlparse.cpp | 1035 +++++++++++------ base/poco/XML/src/xmlrole.c | 115 +- base/poco/XML/src/xmlrole.h | 8 +- base/poco/XML/src/xmltok.c | 412 +++---- base/poco/XML/src/xmltok.h | 66 +- base/poco/XML/src/xmltok_impl.c | 67 +- base/poco/XML/src/xmltok_impl.h | 2 + base/poco/XML/src/xmltok_ns.c | 9 +- base/poco/XML/src/{xcsinc.c => xmlxcs.c} | 48 +- 23 files changed, 1365 insertions(+), 822 deletions(-) create mode 100644 base/poco/XML/src/hash_table.h create mode 100644 base/poco/XML/src/xcs.h rename base/poco/XML/src/{xcsinc.c => xmlxcs.c} (65%) diff --git a/base/poco/XML/CMakeLists.txt b/base/poco/XML/CMakeLists.txt index 0bcbb22dc4925..1d3c8d332f1b1 100644 --- a/base/poco/XML/CMakeLists.txt +++ b/base/poco/XML/CMakeLists.txt @@ -9,6 +9,7 @@ add_library (Poco::XML::Expat ALIAS _poco_xml_expat) target_compile_options (_poco_xml_expat PRIVATE -Wno-cast-qual + -Wno-disabled-macro-expansion -Wno-empty-translation-unit -Wno-extra-semi-stmt -Wno-implicit-fallthrough @@ -49,6 +50,7 @@ target_compile_options (_poco_xml -Wno-unreachable-code -Wno-unreachable-code-break -Wno-unreachable-code-fallthrough + -Wno-unreachable-code-return -Wno-unused-macros -Wno-unused-parameter -Wno-zero-as-null-pointer-constant diff --git a/base/poco/XML/include/Poco/XML/expat.h b/base/poco/XML/include/Poco/XML/expat.h index 7b3c0dd405805..4a57c70fec8dd 100644 --- a/base/poco/XML/include/Poco/XML/expat.h +++ b/base/poco/XML/include/Poco/XML/expat.h @@ -20,6 +20,7 @@ Copyright (c) 2023 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Copyright (c) 2025 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -40,12 +41,14 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_INCLUDED # define Expat_INCLUDED 1 -# include // for uint8_t +# include // for int64_t, uint8_t, uint64_t # include # include "expat_external.h" @@ -137,6 +140,26 @@ enum XML_Error { XML_ERROR_NOT_STARTED, }; +/* Added in 2.9.0. */ +enum XML_Prop_Error { + XML_PROP_ERROR_NONE = 0, // i.e. success + + XML_PROP_ERROR_INVALID_KEY = 1, // i.e. the property is not known at all + XML_PROP_ERROR_INVALID_TYPE = 2, // i.e. the property is known but expects a + // different type (so a different + // getter/setter would need to be called) + XML_PROP_ERROR_INVALID_VALUE = 3, // i.e. the value is invalid (with a setter + // function) or a NULL-pointer (with a + // getter function) + + XML_PROP_ERROR_PARSER_NULL = 4, // i.e. the parser argument is NULL + XML_PROP_ERROR_PARSER_NOT_ROOT = 5, // i.e. the parser is not a root parser + // but a subparser (and a root parser is + // needed) + + /* potentially more error codes upcoming here */ +}; + enum XML_Content_Type { XML_CTYPE_EMPTY = 1, XML_CTYPE_ANY, @@ -754,29 +777,6 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -# ifdef XML_ATTR_INFO -/* Source file byte offsets for the start and end of attribute names and values. - The value indices are exclusive of surrounding quotes; thus in a UTF-8 source - file an attribute value of "blah" will yield: - info->valueEnd - info->valueStart = 4 bytes. -*/ -typedef struct { - XML_Index nameStart; /* Offset to beginning of the attribute name. */ - XML_Index nameEnd; /* Offset after the attribute name's last byte. */ - XML_Index valueStart; /* Offset to beginning of the attribute value. */ - XML_Index valueEnd; /* Offset after the attribute value's last byte. */ -} XML_AttrInfo; - -/* Returns an array of XML_AttrInfo structures for the attribute/value pairs - passed in last call to the XML_StartElementHandler that were specified - in the start-tag rather than defaulted. Each attribute/value pair counts - as 1; thus the number of entries in the array is - XML_GetSpecifiedAttributeCount(parser) / 2. -*/ -XMLPARSEAPI(const XML_AttrInfo *) -XML_GetAttributeInfo(XML_Parser parser); -# endif - /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len may be zero for this call (or any other). @@ -919,7 +919,9 @@ XML_SetParamEntityParsing(XML_Parser parser, Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. DEPRECATED since Expat 2.8.0. + Please use XML_SetHashSalt16Bytes instead. */ +XML_ATTR_DEPRECATED("please use XML_SetHashSalt16Bytes instead") XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); @@ -959,15 +961,24 @@ XML_GetErrorCode(XML_Parser parser); return 0 to indicate an error. Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentLineNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentLineNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentColumnNumber64 instead") XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); +XMLPARSEAPI(uint64_t) XML_GetCurrentColumnNumber64(XML_Parser parser); +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteIndex64 instead") XMLPARSEAPI(XML_Index) XML_GetCurrentByteIndex(XML_Parser parser); +XMLPARSEAPI(int64_t) XML_GetCurrentByteIndex64(XML_Parser parser); /* Return the number of bytes in the current event. Returns 0 if the event is in an internal entity. */ +XML_ATTR_DEPRECATED("please use XML_GetCurrentByteCount64 instead") XMLPARSEAPI(int) XML_GetCurrentByteCount(XML_Parser parser); +XMLPARSEAPI(uint64_t) +XML_GetCurrentByteCount64(XML_Parser parser); /* If XML_CONTEXT_BYTES is >=1, returns the input buffer, sets the integer pointed to by offset to the offset within this buffer @@ -979,8 +990,11 @@ XML_GetCurrentByteCount(XML_Parser parser); NOTE: The character pointer returned should not be used outside the handler that makes the call. */ +XML_ATTR_DEPRECATED("please use XML_GetInputContext64 instead") XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); +XMLPARSEAPI(const char *) +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size); /* For backwards compatibility with previous versions. */ # define XML_GetErrorLineNumber XML_GetCurrentLineNumber @@ -1035,11 +1049,15 @@ enum XML_FeatureEnum { XML_FEATURE_UNICODE_WCHAR_T, XML_FEATURE_DTD, XML_FEATURE_CONTEXT_BYTES, + /* Added in Expat 1.95.5, removed in Expat 2.9.0. */ XML_FEATURE_MIN_SIZE, XML_FEATURE_SIZEOF_XML_CHAR, XML_FEATURE_SIZEOF_XML_LCHAR, + /* Added in Expat 2.0.0. */ XML_FEATURE_NS, + /* Added in Expat 2.0.1. */ XML_FEATURE_LARGE_SIZE, + /* Added in Expat 2.1.0, removed in Expat 2.9.0. */ XML_FEATURE_ATTR_INFO, /* Added in Expat 2.4.0. */ XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, @@ -1061,6 +1079,41 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); +/* Added in Expat 2.9.0. */ +enum XML_Parser_Property { +# if defined(XML_TESTING) + XML_PROP_INVALID = 0, +# endif +# if XML_GE == 1 + XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` + XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` + XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` + XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION = 4, // of type `double` +# endif + XML_PROP_REPARSE_DEFERRAL_ENABLED = 5, // of type `XML_Bool` + /* more properties upcoming here */ +}; + +/* Added in Expat 2.9.0. */ +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value); +XMLPARSEAPI(enum XML_Prop_Error) +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value); + # if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ @@ -1093,8 +1146,8 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 2 +# define XML_MINOR_VERSION 9 +# define XML_MICRO_VERSION 0 # ifdef __cplusplus } diff --git a/base/poco/XML/include/Poco/XML/expat_external.h b/base/poco/XML/include/Poco/XML/expat_external.h index 1c882c3a41ecd..9fb80149acd3a 100644 --- a/base/poco/XML/include/Poco/XML/expat_external.h +++ b/base/poco/XML/include/Poco/XML/expat_external.h @@ -16,6 +16,7 @@ Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov Copyright (c) 2026 Matthew Fernandez + Copyright (c) 2026 Braian Plaku Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,6 +37,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef Expat_External_INCLUDED @@ -123,6 +126,17 @@ # define XML_ATTR_ALLOC_SIZE(x) # endif +/* Marks a function that Expat still provides but that callers should move off + of. */ +# if defined(__clang__) || defined(__GNUC__) +# define XML_ATTR_DEPRECATED(message) \ + __attribute__((__deprecated__(message))) +# elif defined(_MSC_VER) +# define XML_ATTR_DEPRECATED(message) __declspec(deprecated(message)) +# else +# define XML_ATTR_DEPRECATED(message) // empty i.e. no deprecation +# endif + # define XMLPARSEAPI(type) XMLIMPORT type XMLCALL # ifdef __cplusplus @@ -152,6 +166,9 @@ typedef char XML_LChar; # endif /* XML_UNICODE */ # ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# if defined(__clang__) || defined(__GNUC__) +# warning Macro XML_LARGE_SIZE is deprecated, please use the 64bit location API functions instead. +# endif typedef long long XML_Index; typedef unsigned long long XML_Size; # else diff --git a/base/poco/XML/src/ascii.h b/base/poco/XML/src/ascii.h index 3e1f61ec03a80..56c8807aa5746 100644 --- a/base/poco/XML/src/ascii.h +++ b/base/poco/XML/src/ascii.h @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define ASCII_A 0x41 diff --git a/base/poco/XML/src/asciitab.h b/base/poco/XML/src/asciitab.h index ea92a318ffde0..4708a7295bf6c 100644 --- a/base/poco/XML/src/asciitab.h +++ b/base/poco/XML/src/asciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x00 */ BT_NONXML, BT_NONXML, BT_NONXML, BT_NONXML, diff --git a/base/poco/XML/src/fallthrough.h b/base/poco/XML/src/fallthrough.h index cae4c6274da3a..819a42d1d34c1 100644 --- a/base/poco/XML/src/fallthrough.h +++ b/base/poco/XML/src/fallthrough.h @@ -27,6 +27,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef FALLTHROUGH_H diff --git a/base/poco/XML/src/hash_table.h b/base/poco/XML/src/hash_table.h new file mode 100644 index 0000000000000..ee637623c2f04 --- /dev/null +++ b/base/poco/XML/src/hash_table.h @@ -0,0 +1,76 @@ +/* Hash table related internal API + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(HASH_TABLE_H) +# define HASH_TABLE_H 1 + +# include "Poco/XML/expat.h" // for XML_Bool, XML_Parser +# include "internal.h" // for XML_NONTESTING_STATIC + +# include // for size_t + +typedef const XML_Char *KEY; + +typedef struct { + KEY name; +} NAMED; + +typedef struct { + NAMED **v; + unsigned char power; + size_t size; + size_t used; + XML_Parser parser; +} HASH_TABLE; + +typedef struct { + NAMED **p; + NAMED **end; +} HASH_TABLE_ITER; + +XML_NONTESTING_STATIC NAMED *lookupWithLength(XML_Parser parser, + HASH_TABLE *table, KEY name, + size_t nameLen, + size_t createSize); +XML_NONTESTING_STATIC NAMED *lookup(XML_Parser parser, HASH_TABLE *table, + KEY name, size_t createSize); + +XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *table, XML_Parser parser); +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table); +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, + const HASH_TABLE *table); +XML_NONTESTING_STATIC NAMED *hashTableIterNext(HASH_TABLE_ITER *iter); + +XML_NONTESTING_STATIC XML_Bool keyeq(KEY s1, size_t s1len, KEY s2); +XML_NONTESTING_STATIC size_t keylen(KEY s); + +#endif // ! defined(HASH_TABLE_H) diff --git a/base/poco/XML/src/iasciitab.h b/base/poco/XML/src/iasciitab.h index fdadd766afb73..652b3c03e02da 100644 --- a/base/poco/XML/src/iasciitab.h +++ b/base/poco/XML/src/iasciitab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* Like asciitab.h, except that 0xD has code BT_S rather than BT_CR */ diff --git a/base/poco/XML/src/internal.h b/base/poco/XML/src/internal.h index f9ffbc72cf810..d0465449a3991 100644 --- a/base/poco/XML/src/internal.h +++ b/base/poco/XML/src/internal.h @@ -6,13 +6,6 @@ The following calling convention macros are defined for frequently called functions: - FASTCALL - Used for those internal functions that have a simple - body and a low number of arguments and local variables. - - PTRCALL - Used for functions called though function pointers. - - PTRFASTCALL - Like PTRCALL, but for low number of arguments. - inline - Used for selected internal functions for which inlining may improve performance on some platforms. @@ -33,6 +26,9 @@ Copyright (c) 2019 David Loffredo Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2026 Matthew Wozniczka + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Florian Schmaus Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -53,117 +49,83 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#if defined(__GNUC__) && defined(__i386__) && ! defined(__MINGW32__) -/* We'll use this version by default only where we know it helps. - - regparm() generates warnings on Solaris boxes. See SF bug #692878. - - Instability reported with egcs on a RedHat Linux 7.3. - Let's comment out: - #define FASTCALL __attribute__((stdcall, regparm(3))) - and let's try this: -*/ -# define FASTCALL __attribute__((regparm(3))) -# define PTRFASTCALL __attribute__((regparm(3))) -#endif -/* Using __fastcall seems to have an unexpected negative effect under - MS VC++, especially for function pointers, so we won't use it for - now on that platform. It may be reconsidered for a future release - if it can be made more effective. - Likely reason: __fastcall on Windows is like stdcall, therefore - the compiler cannot perform stack optimizations for call clusters. + SPDX-License-Identifier: MIT */ -/* Make sure all of these are defined if they aren't already. */ - -#ifndef FASTCALL -# define FASTCALL -#endif - -#ifndef PTRCALL -# define PTRCALL -#endif - -#ifndef PTRFASTCALL -# define PTRFASTCALL -#endif +#if ! defined(INTERNAL_H) +# define INTERNAL_H 1 -#ifndef XML_MIN_SIZE -# if ! defined(__cplusplus) && ! defined(inline) +# if ! defined(inline) # ifdef __GNUC__ # define inline __inline # endif /* __GNUC__ */ # endif -#endif /* XML_MIN_SIZE */ -#ifdef __cplusplus -# define inline inline -#else # ifndef inline # define inline # endif -#endif -#include // ULONG_MAX -#include // size_t - -#if defined(_WIN32) \ - && (! defined(__USE_MINGW_ANSI_STDIO) \ - || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) -# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" -# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" -# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" -# else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# if ! defined(XML_NONTESTING_STATIC) +# if defined(XML_TESTING) +# define XML_NONTESTING_STATIC // empty i.e. not static +# else +# define XML_NONTESTING_STATIC static +# endif # endif -#else -# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" -# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" -# if ! defined(ULONG_MAX) -# error Compiler did not define ULONG_MAX for us -# elif ULONG_MAX == 18446744073709551615u // 2^64-1 -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" -# elif defined(__wasm32__) // 32bit mode Emscripten or WASI SDK -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "zu" + +# include // ULONG_MAX +# include // size_t + +# if defined(_WIN32) \ + && (! defined(__USE_MINGW_ANSI_STDIO) \ + || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif # else -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# include // PRIdPTR, PRIuPTR +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR # endif -#endif -#ifndef UNUSED_P -# define UNUSED_P(p) (void)p -#endif +# ifndef UNUSED_P +# define UNUSED_P(p) (void)p +# endif /* NOTE BEGIN If you ever patch these defaults to greater values for non-attack XML payload in your environment, please file a bug report with libexpat. Thank you! */ -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ - 100.0f -#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ - 8388608 // 8 MiB, 2^23 +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +# define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 -#define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f -#define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ - 67108864 // 64 MiB, 2^26 +# define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f +# define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ + 67108864 // 64 MiB, 2^26 // NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would // have to take sizeof(long double) into account -#define EXPAT_MALLOC_ALIGNMENT sizeof(long long) // largest parser (sub)member -#define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) +union expat_align { + long long l; + void *p; +}; +# define EXPAT_MALLOC_ALIGNMENT sizeof(union expat_align) +# define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) /* NOTE END */ -#include "Poco/XML/expat.h" // so we can use type XML_Parser below +# include "Poco/XML/expat.h" // so we can use type XML_Parser below #ifdef __cplusplus extern "C" { @@ -172,24 +134,29 @@ extern "C" { void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); -#if defined(XML_GE) && XML_GE == 1 +# if defined(XML_GE) && XML_GE == 1 unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); const char *unsignedCharToPrintable(unsigned char c); -#endif +# endif extern -#if ! defined(XML_TESTING) +# if ! defined(XML_TESTING) const -#endif +# endif XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c -#if defined(XML_TESTING) +# if defined(XML_TESTING) + +int xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt); + void *expat_malloc(XML_Parser parser, size_t size, int sourceLine); void expat_free(XML_Parser parser, void *ptr, int sourceLine); void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine); extern unsigned int g_bytesScanned; // used for testing only -#endif +# endif #ifdef __cplusplus } #endif + +#endif // not defined INTERNAL_H diff --git a/base/poco/XML/src/latin1tab.h b/base/poco/XML/src/latin1tab.h index 59cd962cbdec3..48bbf4c7fd6ca 100644 --- a/base/poco/XML/src/latin1tab.h +++ b/base/poco/XML/src/latin1tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_OTHER, BT_OTHER, BT_OTHER, BT_OTHER, diff --git a/base/poco/XML/src/nametab.h b/base/poco/XML/src/nametab.h index 616bc29662bb3..1047453942a65 100644 --- a/base/poco/XML/src/nametab.h +++ b/base/poco/XML/src/nametab.h @@ -28,6 +28,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ static const unsigned namingBitmap[] = { diff --git a/base/poco/XML/src/siphash.h b/base/poco/XML/src/siphash.h index 906a37613e9e1..cca88c11fd4f1 100644 --- a/base/poco/XML/src/siphash.h +++ b/base/poco/XML/src/siphash.h @@ -8,6 +8,8 @@ * * 1. https://www.131002.net/siphash/siphash24.c * 2. https://www.131002.net/siphash/ + * + * SPDX-License-Identifier: CC0-1.0 * -------------------------------------------------------------------------- * HISTORY: * diff --git a/base/poco/XML/src/utf8tab.h b/base/poco/XML/src/utf8tab.h index 825be4eb1bbc4..60848da01d8b7 100644 --- a/base/poco/XML/src/utf8tab.h +++ b/base/poco/XML/src/utf8tab.h @@ -30,6 +30,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ /* 0x80 */ BT_TRAIL, BT_TRAIL, BT_TRAIL, BT_TRAIL, diff --git a/base/poco/XML/src/xcs.h b/base/poco/XML/src/xcs.h new file mode 100644 index 0000000000000..1e63a9b1de2f6 --- /dev/null +++ b/base/poco/XML/src/xcs.h @@ -0,0 +1,52 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(EXPAT_XCS_H) + +# include // size_t +# include "Poco/XML/expat_external.h" // for XML_Char + +// `xcs.c` is compiled as C while `xmlparse.cpp` is compiled as C++, so the +// declarations need C linkage to match the definitions. +#ifdef __cplusplus +extern "C" { +#endif + +extern size_t xcslen(const XML_Char *s); + +extern int xcscmp(const XML_Char *a, const XML_Char *b); +extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif // ! defined(EXPAT_XCS_H) diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index f019aef201c95..001de556c8912 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -1,4 +1,4 @@ -/* 5de44e6750c6cc78818f06ed552f522a1241df0299395250e1792cb339389daf (2.8.2+) +/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -50,6 +50,14 @@ Copyright (c) 2026 Nick Begg Copyright (c) 2026 Kartik Kenchi Copyright (c) 2026 Haris Hussain + Copyright (c) 2026 Evgeny Kotkov + Copyright (c) 2026 Darren Carreras + Copyright (c) 2026 Alberto Maschietto + Copyright (c) 2026 Zeyou Liu + Copyright (c) 2026 Stan Ulbrych + Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Filippo Tedeschi + Copyright (c) 2026 Junki Lee Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -70,6 +78,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #define XML_BUILDING_EXPAT 1 @@ -93,12 +103,13 @@ #include #include /* memset(), memcpy() */ #include -#include /* INT_MAX, LLONG_MAX, LONG_MAX, UINT_MAX */ +#include /* INT_MAX, UINT_MAX */ #include /* fprintf */ #include /* getenv */ -#include /* SIZE_MAX, uintptr_t */ +#include /* SIZE_MAX, UINT64_MAX, uint64_t, uintptr_t */ #include /* isnan */ #include +#include /* wcsncmp */ #define EXPAT_POCO 1 @@ -122,7 +133,7 @@ #include "ascii.h" #include "Poco/XML/expat.h" #include "siphash.h" -#include "xcsinc.c" +#include "xcs.h" #if defined(HAVE_ARC4RANDOM) # include "random_arc4random.h" @@ -217,12 +228,6 @@ typedef char ICHAR; #endif -#ifdef XML_LARGE_SIZE -# define XML_INDEX_MAX LLONG_MAX -#else -# define XML_INDEX_MAX LONG_MAX -#endif - /* Round up n to be a multiple of sz, where sz is a power of 2. */ #define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) @@ -232,25 +237,10 @@ typedef char ICHAR; #define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) #include "internal.h" +#include "hash_table.h" #include "xmltok.h" #include "xmlrole.h" -typedef const XML_Char *KEY; - -typedef struct { - KEY name; -} NAMED; - -typedef struct { - NAMED **v; - unsigned char power; - size_t size; - size_t used; - XML_Parser parser; -} HASH_TABLE; - -static size_t keylen(KEY s); - static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); /* For probing (after a collision) we need a step size relative prime @@ -266,11 +256,6 @@ static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key); #define PROBE_STEP(hash, mask, power) \ ((unsigned char)((SECOND_HASH(hash, mask, power)) | 1)) -typedef struct { - NAMED **p; - NAMED **end; -} HASH_TABLE_ITER; - #define INIT_TAG_BUF_SIZE 32 /* must be a multiple of sizeof(XML_Char) */ #define INIT_DATA_BUF_SIZE 1024 #define INIT_ATTS_SIZE 16 @@ -339,13 +324,13 @@ typedef struct { const XML_Char *base; const XML_Char *publicId; const XML_Char *notation; - XML_Bool open; - XML_Bool hasMore; /* true if entity has not been completely processed */ - /* An entity can be open while being already completely processed (hasMore == - XML_FALSE). The reason is the delayed closing of entities until their inner + bool open; + bool hasMore; /* true if entity has not been completely processed */ + /* An entity can be open while being already completely processed (!hasMore). + The reason is the delayed closing of entities until their inner entities are processed and closed */ - XML_Bool is_param; - XML_Bool is_internal; /* true if declared in internal subset outside PE */ + bool is_param; + bool is_internal; /* true if declared in internal subset outside PE */ } ENTITY; typedef struct { @@ -380,8 +365,8 @@ typedef struct { typedef struct attribute_id { XML_Char *name; PREFIX *prefix; - XML_Bool maybeTokenized; - XML_Bool xmlns; + bool maybeTokenized; + bool xmlns; } ATTRIBUTE_ID; typedef struct { @@ -390,6 +375,22 @@ typedef struct { const XML_Char *value; } DEFAULT_ATTRIBUTE; +// This structure allows mapping attribute names to instances of +// `DEFAULT_ATTRIBUTE`. +typedef struct { + // Member `name` goes first to make this structure compatible with structure + // `NAMED` (further up), which is needed to support use of structure + // `NAME_AND_DEFAULT_ATTRIBUTE` in a hash table as implemented by function + // `lookup` (further down). + const XML_Char *name; + // We would store a `DEFAULT_ATTRIBUTE *` here but the backing array + // can be reallocated which would invalidate the pointer. Using an index + // into the array instead, avoids that problem. + size_t attIndex; + // This is set to `false` by function `lookup`. + bool initialized; +} NAME_AND_DEFAULT_ATTRIBUTE; + typedef struct { unsigned long version; unsigned long hash; @@ -403,7 +404,7 @@ typedef struct { size_t nDefaultAtts; size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; - HASH_TABLE defaultAttsNames; + HASH_TABLE defaultAttForName; } ELEMENT_TYPE; typedef struct { @@ -421,12 +422,12 @@ typedef struct { XML_Bool standalone; #ifdef XML_DTD /* indicates if external PE has been read */ - XML_Bool paramEntityRead; + bool paramEntityRead; HASH_TABLE paramEntities; #endif /* XML_DTD */ PREFIX defaultPrefix; /* === scaffolding for building content model === */ - XML_Bool in_eldecl; + bool in_eldecl; CONTENT_SCAFFOLD *scaffold; unsigned contentStringLen; unsigned scaffSize; @@ -448,7 +449,7 @@ typedef struct open_internal_entity { struct open_internal_entity *next; ENTITY *entity; int startTagLevel; - XML_Bool betweenDecl; /* WFC: PE Between Declarations */ + bool betweenDecl; /* WFC: PE Between Declarations */ enum EntityType type; } OPEN_INTERNAL_ENTITY; @@ -470,8 +471,8 @@ typedef struct accounting { } ACCOUNTING; typedef struct MALLOC_TRACKER { - XmlBigCount bytesAllocated; - XmlBigCount peakBytesAllocated; // updated live only for debug level >=2 + size_t bytesAllocated; + size_t peakBytesAllocated; // updated live only for debug level >=2 unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 XmlBigCount activationThresholdBytes; @@ -485,8 +486,8 @@ typedef struct entity_stats { } ENTITY_STATS; #endif /* XML_GE == 1 */ -typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, - const char *end, const char **endPtr); +typedef enum XML_Error Processor(XML_Parser parser, const char *start, + const char *end, const char **endPtr); static Processor prologProcessor; static Processor prologInitProcessor; @@ -518,7 +519,7 @@ static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, XML_Bool haveMore, XML_Bool allowClosingDoctype, enum XML_Account account); static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, - XML_Bool betweenDecl, enum EntityType type); + bool betweenDecl, enum EntityType type); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, @@ -578,7 +579,7 @@ static void reportDefault(XML_Parser parser, const ENCODING *enc, static const XML_Char *getContext(XML_Parser parser); static XML_Bool setContext(XML_Parser parser, const XML_Char *context); -static void FASTCALL normalizePublicId(XML_Char *s); +static void normalizePublicId(XML_Char *s); static DTD *dtdCreate(XML_Parser parser); /* do not call if m_parentParser != NULL */ @@ -588,34 +589,24 @@ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); -static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, - size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); -static void FASTCALL hashTableClear(HASH_TABLE *table); -static void FASTCALL hashTableDestroy(HASH_TABLE *table); -static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, - const HASH_TABLE *table); -static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); - -static void FASTCALL poolInit(STRING_POOL *pool, XML_Parser parser); -static void FASTCALL poolClear(STRING_POOL *pool); -static void FASTCALL poolDestroy(STRING_POOL *pool); + +static void poolInit(STRING_POOL *pool, XML_Parser parser); +static void poolClear(STRING_POOL *pool); +static void poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); -static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); -static bool FASTCALL poolGrowUntil(STRING_POOL *pool, size_t needed); -static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, - const XML_Char *s); -static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, - const XML_Char *s); +static XML_Bool poolGrow(STRING_POOL *pool); +static bool poolGrowUntil(STRING_POOL *pool, size_t needed); +static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s); +static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool, + const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n); -static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, - const XML_Char *s); +static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s); -static int FASTCALL nextScaffoldPart(XML_Parser parser); +static int nextScaffoldPart(XML_Parser parser); static XML_Content *build_model(XML_Parser parser); static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); @@ -664,6 +655,25 @@ static bool poolAppendChar(STRING_POOL *pool, XML_Char c); static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); +#if XML_GE == 1 +static enum XML_Prop_Error setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +static enum XML_Prop_Error setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold(XML_Parser parser, + unsigned long long activationThresholdBytes); +#endif /* XML_GE == 1 */ + +static enum XML_Prop_Error setReparseDeferralEnabled(XML_Parser parser, + XML_Bool enabled); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) @@ -725,7 +735,7 @@ struct XML_ParserStruct { char *m_bufferEnd; // past last character to be parsed const char *m_bufferLim; // allocated end of m_buffer - XML_Index m_parseEndByteIndex; + uint64_t m_parseEndByteIndex; const char *m_parseEndPtr; size_t m_partialTokenBytesBefore; /* used in heuristic to avoid O(n^2) */ XML_Bool m_reparseDeferralEnabled; @@ -764,6 +774,8 @@ struct XML_ParserStruct { void *m_unknownEncodingMem; void *m_unknownEncodingData; void *m_unknownEncodingHandlerData; + // Application callback invoked by callUnknownEncodingConvert. + int(XMLCALL *m_unknownEncodingConvert)(void *, const char *); void(XMLCALL *m_unknownEncodingRelease)(void *); PROLOG_STATE m_prologState; Processor *m_processor; @@ -801,9 +813,6 @@ struct XML_ParserStruct { NS_ATT *m_nsAtts; unsigned long m_nsAttsVersion; unsigned char m_nsAttsPower; -#ifdef XML_ATTR_INFO - XML_AttrInfo *m_attInfo; -#endif POSITION m_position; STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; @@ -840,30 +849,30 @@ struct XML_ParserStruct { #if XML_GE == 1 static void -expat_heap_stat(XML_Parser rootParser, char operator, XmlBigCount absDiff, - XmlBigCount newTotal, XmlBigCount peakTotal, int sourceLine) { +expat_heap_stat(XML_Parser rootParser, char operator, size_t absDiff, + size_t newTotal, size_t peakTotal, int sourceLine) { // NOTE: This can be +infinity or -nan const float amplification = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; fprintf( stderr, - "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_ULL( - "10") " to " EXPAT_FMT_ULL("10") " (" EXPAT_FMT_ULL("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_SIZE_T( + "10") " to " EXPAT_FMT_SIZE_T("10") " (" EXPAT_FMT_SIZE_T("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, absDiff, newTotal, peakTotal, (double)amplification, sourceLine); } static bool -expat_heap_increase_tolerable(XML_Parser rootParser, XmlBigCount increase, +expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, int sourceLine) { assert(rootParser != NULL); assert(increase > 0); - XmlBigCount newTotal = 0; + size_t newTotal = 0; bool tolerable = true; // Detect integer overflow - if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated < increase) { + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { tolerable = false; } else { newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; @@ -903,8 +912,7 @@ expat_malloc(XML_Parser parser, size_t size, int sourceLine) { const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; - if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated - < bytesToAllocate) { + if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < bytesToAllocate) { return NULL; // i.e. signal integer overflow as out-of-memory } @@ -1014,9 +1022,10 @@ expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { } } - // NOTE: Integer overflow detection has already been done for us - // by expat_heap_increase_tolerable(..) above - assert(SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING >= size); + // Detect and prevent integer overflow + if (size > SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING) { + return NULL; + } // Actually allocate mallocedPtr = parser->m_mem.realloc_fcn( @@ -1028,8 +1037,7 @@ expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { // Update accounting if (isIncrease) { - assert((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated - >= absDiff); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= absDiff); rootParser->m_alloc_tracker.bytesAllocated += absDiff; } else { // i.e. decrease assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); @@ -1203,6 +1211,25 @@ isCalledFromInsideHandler(XML_Parser parser) { return parser->m_handlerCallDepth > 0; } +static void +callUnknownEncodingRelease(XML_Parser parser) { + beforeHandler(parser); + parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); + afterHandler(parser); + parser->m_unknownEncodingRelease = NULL; + parser->m_unknownEncodingData = NULL; +} + +static int XMLCALL +callUnknownEncodingConvert(void *data, const char *p) { + XML_Parser parser = (XML_Parser)data; + beforeHandler(parser); + const int result + = parser->m_unknownEncodingConvert(parser->m_unknownEncodingData, p); + afterHandler(parser); + return result; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1403,20 +1430,9 @@ parserCreate(const XML_Char *encodingName, FREE(parser, parser); return NULL; } -#ifdef XML_ATTR_INFO - parser->m_attInfo = (XML_AttrInfo*)MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (parser->m_attInfo == NULL) { - FREE(parser, parser->m_atts); - FREE(parser, parser); - return NULL; - } -#endif parser->m_dataBuf = (XML_Char*)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1429,9 +1445,6 @@ parserCreate(const XML_Char *encodingName, if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser); return NULL; } @@ -1550,6 +1563,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_inheritedBindings = NULL; parser->m_nSpecifiedAtts = 0; parser->m_unknownEncodingMem = NULL; + parser->m_unknownEncodingConvert = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; @@ -1579,7 +1593,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { } /* moves list of bindings to m_freeBindingList */ -static void FASTCALL +static void moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { while (bindings) { BINDING *b = bindings; @@ -1609,6 +1623,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { if (parser->m_parentParser) return XML_FALSE; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* move m_tagStack to m_freeTagList */ tStk = parser->m_tagStack; while (tStk) { @@ -1629,14 +1647,16 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); poolClear(&parser->m_tempPool); poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(parser->m_dtd, parser); + if (encodingName && ! parser->m_protocolEncodingName) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + return XML_FALSE; + } return XML_TRUE; } @@ -1852,7 +1872,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, return parser; } -static void FASTCALL +static void destroyBindings(BINDING *bindings, XML_Parser parser) { for (;;) { BINDING *b = bindings; @@ -1869,6 +1889,10 @@ XML_ParserFree(XML_Parser parser) { TAG *tagList; if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; + // The application-defined release callback may access the parser, so it + // must run before any parser state is freed. + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; for (;;) { @@ -1929,9 +1953,6 @@ XML_ParserFree(XML_Parser parser) { #endif /* XML_DTD */ dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); FREE(parser, parser->m_atts); -#ifdef XML_ATTR_INFO - FREE(parser, parser->m_attInfo); -#endif FREE(parser, parser->m_groupConnector); // NOTE: We are avoiding FREE(..) here because parser->m_buffer // is not being allocated with MALLOC(..) but with plain @@ -1940,8 +1961,6 @@ XML_ParserFree(XML_Parser parser) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); - if (parser->m_unknownEncodingRelease) - parser->m_unknownEncodingRelease(parser->m_unknownEncodingData); FREE(parser, parser); } @@ -2022,15 +2041,6 @@ XML_GetIdAttributeIndex(XML_Parser parser) { return parser->m_idAttIndex; } -#ifdef XML_ATTR_INFO -const XML_AttrInfo *XMLCALL -XML_GetAttributeInfo(XML_Parser parser) { - if (parser == NULL) - return NULL; - return parser->m_attInfo; -} -#endif - void XMLCALL XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { @@ -2251,9 +2261,10 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } -// DEPRECATED since Expat 2.8.0. -int XMLCALL -XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { +/* The body of XML_SetHashSalt, so that Expat's own tests can reach it + without tripping the deprecation of the public function. */ +XML_NONTESTING_STATIC int +xmlSetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) return 0; @@ -2279,6 +2290,12 @@ XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { return 1; } +// DEPRECATED since Expat 2.8.0. +int XMLCALL +XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + return xmlSetHashSalt(parser, hash_salt); +} + XML_Bool XMLCALL XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { if (parser == NULL) @@ -2335,7 +2352,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2452,8 +2469,13 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_PARSING; } + if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd)) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } + // Detect and avoid integer overflow - if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2711,35 +2733,57 @@ XML_GetErrorCode(XML_Parser parser) { return parser->m_errorCode; } -XML_Index XMLCALL -XML_GetCurrentByteIndex(XML_Parser parser) { +int64_t XMLCALL +XML_GetCurrentByteIndex64(XML_Parser parser) { if (parser == NULL) return -1; - if (parser->m_eventPtr) - return (XML_Index)(parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - parser->m_eventPtr)); + if (parser->m_eventPtr) { + return (int64_t)(parser->m_parseEndByteIndex + - (parser->m_parseEndPtr - parser->m_eventPtr)); + } return -1; } -int XMLCALL -XML_GetCurrentByteCount(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +XML_Index XMLCALL +XML_GetCurrentByteIndex(XML_Parser parser) { + // NOTE: XML_Index is known to wrap around for >2 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Index)XML_GetCurrentByteIndex64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentByteCount64(XML_Parser parser) { if (parser == NULL) return 0; - if (parser->m_eventEndPtr && parser->m_eventPtr) - return (int)(parser->m_eventEndPtr - parser->m_eventPtr); + if (parser->m_eventEndPtr && parser->m_eventPtr) { + return parser->m_eventEndPtr - parser->m_eventPtr; + } return 0; } +// DEPRECATED since Expat 2.9.0. +int XMLCALL +XML_GetCurrentByteCount(XML_Parser parser) { + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (int)XML_GetCurrentByteCount64(parser); +} + const char *XMLCALL -XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +XML_GetInputContext64(XML_Parser parser, int64_t *offset, uint64_t *size) { #if XML_CONTEXT_BYTES > 0 if (parser == NULL) return NULL; if (parser->m_eventPtr && parser->m_buffer) { if (offset != NULL) - *offset = (int)(parser->m_eventPtr - parser->m_buffer); + *offset = parser->m_eventPtr - parser->m_buffer; if (size != NULL) - *size = (int)(parser->m_bufferEnd - parser->m_buffer); + *size = parser->m_bufferEnd - parser->m_buffer; return parser->m_buffer; } #else @@ -2750,8 +2794,41 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { return NULL; } -XML_Size XMLCALL -XML_GetCurrentLineNumber(XML_Parser parser) { +// DEPRECATED since Expat 2.9.0. +const char *XMLCALL +XML_GetInputContext(XML_Parser parser, int *offset, int *size) { +#if XML_CONTEXT_BYTES > 0 + if (parser == NULL) + return NULL; + + int64_t offset64; + uint64_t size64; + + const char *const buffer = XML_GetInputContext64(parser, &offset64, &size64); + + if (buffer == NULL) + return NULL; + + // NOTE: int is known to wrap around for >2 GiB content. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + if (offset != NULL) + *offset = (int)offset64; + + if (size != NULL) + *size = (int)size64; + + return buffer; +#else + (void)parser; + (void)offset; + (void)size; +#endif /* XML_CONTEXT_BYTES > 0 */ + return NULL; +} + +uint64_t XMLCALL +XML_GetCurrentLineNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2762,8 +2839,19 @@ XML_GetCurrentLineNumber(XML_Parser parser) { return parser->m_position.lineNumber + 1; } +// DEPRECATED since Expat 2.9.0. XML_Size XMLCALL -XML_GetCurrentColumnNumber(XML_Parser parser) { +XML_GetCurrentLineNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentLineNumber64(parser); +} + +uint64_t XMLCALL +XML_GetCurrentColumnNumber64(XML_Parser parser) { if (parser == NULL) return 0; if (parser->m_eventPtr && parser->m_eventPtr >= parser->m_positionPtr) { @@ -2774,6 +2862,17 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { return parser->m_position.columnNumber; } +// DEPRECATED since Expat 2.9.0. +XML_Size XMLCALL +XML_GetCurrentColumnNumber(XML_Parser parser) { + // NOTE: XML_Size is known to wrap around for >4 GiB content + // on 32bit machines and 64bit Windows, unless (non-default and + // uncommon) XML_LARGE_SIZE is defined. + // That's a bug and it only lives on because we cannot break + // ABI compatibility of public API. + return (XML_Size)XML_GetCurrentColumnNumber64(parser); +} + void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { if (parser == NULL) @@ -2817,7 +2916,7 @@ XML_MemFree(XML_Parser parser, void *ptr) { void XMLCALL XML_DefaultCurrent(XML_Parser parser) { - if (parser == NULL) + if (parser == NULL || ! isCalledFromInsideHandler(parser)) return; if (parser->m_defaultHandler) { if (parser->m_openInternalEntities) @@ -2988,18 +3087,12 @@ XML_GetFeatureList(void) { {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), XML_CONTEXT_BYTES}, #endif -#ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, -#endif #ifdef XML_NS {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif -#ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, -#endif #if XML_GE == 1 /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ @@ -3026,59 +3119,294 @@ XML_GetFeatureList(void) { } #if XML_GE == 1 +static enum XML_Prop_Error +setBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; +} + XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionMaximumAmplification( XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; - } - parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; - return XML_TRUE; + return (setBillionLaughsAttackProtectionMaximumAmplification( + parser, maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; - } - parser->m_accounting.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + return (setBillionLaughsAttackProtectionActivationThreshold( + parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + if (isnan(maximumAmplificationFactor) || (maximumAmplificationFactor < 1.0f)) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, float maximumAmplificationFactor) { - if ((parser == NULL) || (parser->m_parentParser != NULL) - || isnan(maximumAmplificationFactor) - || (maximumAmplificationFactor < 1.0f)) { - return XML_FALSE; - } - parser->m_alloc_tracker.maximumAmplificationFactor - = maximumAmplificationFactor; - return XML_TRUE; + return (setAllocTrackerMaximumAmplification(parser, + maximumAmplificationFactor) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +static enum XML_Prop_Error +setAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (parser->m_parentParser != NULL) + return XML_PROP_ERROR_PARSER_NOT_ROOT; + + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + + return XML_PROP_ERROR_NONE; } XML_Bool XMLCALL XML_SetAllocTrackerActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes) { - if ((parser == NULL) || (parser->m_parentParser != NULL)) { - return XML_FALSE; - } - parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; - return XML_TRUE; + return (setAllocTrackerActivationThreshold(parser, activationThresholdBytes) + == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; } #endif /* XML_GE == 1 */ +static enum XML_Prop_Error +setReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (enabled != XML_TRUE && enabled != XML_FALSE) + return XML_PROP_ERROR_INVALID_VALUE; + + parser->m_reparseDeferralEnabled = enabled; + + return XML_PROP_ERROR_NONE; +} + XML_Bool XMLCALL XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled) { - if (parser != NULL && (enabled == XML_TRUE || enabled == XML_FALSE)) { - parser->m_reparseDeferralEnabled = enabled; - return XML_TRUE; + return (setReparseDeferralEnabled(parser, enabled) == XML_PROP_ERROR_NONE) + ? XML_TRUE + : XML_FALSE; +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return setReparseDeferralEnabled(parser, value); + default: + return XML_PROP_ERROR_INVALID_KEY; } - return XML_FALSE; + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + return setAllocTrackerMaximumAmplification(parser, (float)value); + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return setBillionLaughsAttackProtectionMaximumAmplification(parser, + (float)value); + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_SetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + +#if XML_GE == 0 + UNUSED_P(value); +#endif + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + return setAllocTrackerActivationThreshold(parser, + (unsigned long long)value); + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + return setBillionLaughsAttackProtectionActivationThreshold( + parser, (unsigned long long)value); + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + assert(0 && "considered unreachable"); +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyBool(XML_Parser parser, enum XML_Parser_Property property, + XML_Bool *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + return XML_PROP_ERROR_INVALID_TYPE; +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + *value = parser->m_reparseDeferralEnabled; + break; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyDouble(XML_Parser parser, enum XML_Parser_Property property, + double *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_accounting.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + *value = (double)parser->m_alloc_tracker.maximumAmplificationFactor; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; +} + +enum XML_Prop_Error XMLCALL +XML_GetPropertyUInt64(XML_Parser parser, enum XML_Parser_Property property, + uint64_t *value) { + if (parser == NULL) + return XML_PROP_ERROR_PARSER_NULL; + + if (value == NULL) + return XML_PROP_ERROR_INVALID_VALUE; + + switch (property) { +#if XML_GE == 1 + case XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD: + *value = parser->m_accounting.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD: + *value = parser->m_alloc_tracker.activationThresholdBytes; + break; + case XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION: + case XML_PROP_BILLION_LAUGHS_MAXIMUM_AMPLIFICATION: +#endif /* XML_GE == 1 */ + case XML_PROP_REPARSE_DEFERRAL_ENABLED: + return XML_PROP_ERROR_INVALID_TYPE; + default: + return XML_PROP_ERROR_INVALID_KEY; + } + + return XML_PROP_ERROR_NONE; } /* Initially tag->rawName always points into the parse buffer; @@ -3135,7 +3463,7 @@ storeRawNames(XML_Parser parser) { return XML_TRUE; } -static enum XML_Error PTRCALL +static enum XML_Error contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent( @@ -3149,7 +3477,7 @@ contentProcessor(XML_Parser parser, const char *start, const char *end, return result; } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = initializeEncoding(parser); @@ -3159,7 +3487,7 @@ externalEntityInitProcessor(XML_Parser parser, const char *start, return externalEntityInitProcessor2(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor2(XML_Parser parser, const char *start, const char *end, const char **endPtr) { const char *next = start; /* XmlContentTok doesn't always set the last arg */ @@ -3204,7 +3532,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, return externalEntityInitProcessor3(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *end, const char **endPtr) { int tok; @@ -3256,7 +3584,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, return externalEntityContentProcessor(parser, start, end, endPtr); } -static enum XML_Error PTRCALL +static enum XML_Error externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -3420,9 +3748,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return result; } else if (parser->m_externalEntityRefHandler) { const XML_Char *context; - entity->open = XML_TRUE; + entity->open = true; context = getContext(parser); - entity->open = XML_FALSE; + entity->open = false; if (! context) return XML_ERROR_NO_MEMORY; beforeHandler(parser); @@ -3847,8 +4175,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; - if (! elementType->defaultAttsNames.parser) - hashTableInit(&(elementType->defaultAttsNames), parser); + if (! elementType->defaultAttForName.parser) + hashTableInit(&(elementType->defaultAttForName), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -3891,21 +4219,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; -#ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. */ - if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } - - XML_AttrInfo *const temp2 = REALLOC( - parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) { - parser->m_attsSize = oldAttsSize; - return XML_ERROR_NO_MEMORY; - } - parser->m_attInfo = temp2; -#endif if (n > oldAttsSize) { /* Detect and prevent integer overflow. */ if (n > (size_t)INT_MAX) @@ -3918,25 +4231,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; -#ifdef XML_ATTR_INFO - XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; -#endif /* add the name and value to the attribute list */ ATTRIBUTE_ID *attId = getAttributeId(parser, enc, currAtt->name, currAtt->name + XmlNameLength(enc, currAtt->name)); if (! attId) return XML_ERROR_NO_MEMORY; -#ifdef XML_ATTR_INFO - currAttInfo->nameStart - = parser->m_parseEndByteIndex - (parser->m_parseEndPtr - currAtt->name); - currAttInfo->nameEnd - = currAttInfo->nameStart + XmlNameLength(enc, currAtt->name); - currAttInfo->valueStart = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valuePtr); - currAttInfo->valueEnd = parser->m_parseEndByteIndex - - (parser->m_parseEndPtr - currAtt->valueEnd); -#endif /* Detect duplicate attributes by their QNames. This does not work when namespace processing is turned on and different prefixes for the same namespace are used. For this case we have a check further down. @@ -3953,11 +4253,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - for (size_t j = 0; j < nDefaultAtts; j++) { - if (attId == elementType->defaultAtts[j].id) { - isCdata = elementType->defaultAtts[j].isCdata; - break; - } + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(elementType->defaultAttForName), attId->name, 0); + if (nameAndDefaultAttribute != NULL) { + assert(nameAndDefaultAttribute->attIndex < elementType->nDefaultAtts); + const DEFAULT_ATTRIBUTE *const att + = elementType->defaultAtts + nameAndDefaultAttribute->attIndex; + isCdata = att->isCdata; } } @@ -4048,8 +4351,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, unsigned int nsAttsSize = 1u << parser->m_nsAttsPower; unsigned char oldNsAttsPower = parser->m_nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ - if ((nPrefixes << 1) - >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ + if (parser->m_nsAttsPower == 0 + || (nPrefixes >> (parser->m_nsAttsPower - 1))) { /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4507,6 +4810,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, /* Detect and prevent integer overflow */ if (len > SIZE_MAX - EXPAND_SPARE || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { + FREE(parser, b); return XML_ERROR_NO_MEMORY; } @@ -4544,7 +4848,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, /* The idea here is to avoid using stack for each CDATA section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doCdataSection( @@ -4710,7 +5014,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, /* The idea here is to avoid using stack for each IGNORE section when the whole file is parsed with one call. */ -static enum XML_Error PTRCALL +static enum XML_Error ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result @@ -4948,30 +5252,39 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { const int status = parser->m_unknownEncodingHandler( parser->m_unknownEncodingHandlerData, encodingName, &info); afterHandler(parser); + + parser->m_unknownEncodingRelease = info.release; + parser->m_unknownEncodingData = info.data; + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = (void*)MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { - if (info.release) - info.release(info.data); + if (parser->m_unknownEncodingRelease) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; return XML_ERROR_NO_MEMORY; } + parser->m_unknownEncodingConvert = info.convert; enc = (parser->m_ns ? XmlInitUnknownEncodingNS : XmlInitUnknownEncoding)( - parser->m_unknownEncodingMem, info.map, info.convert, info.data); + parser->m_unknownEncodingMem, info.map, + info.convert ? callUnknownEncodingConvert : NULL, parser); if (enc) { - parser->m_unknownEncodingData = info.data; - parser->m_unknownEncodingRelease = info.release; parser->m_encoding = enc; return XML_ERROR_NONE; } + parser->m_unknownEncodingConvert = NULL; } - if (info.release != NULL) - info.release(info.data); + if (parser->m_unknownEncodingRelease != NULL) + callUnknownEncodingRelease(parser); + else + parser->m_unknownEncodingData = NULL; } return XML_ERROR_UNKNOWN_ENCODING; } -static enum XML_Error PTRCALL +static enum XML_Error prologInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -4983,7 +5296,7 @@ prologInitProcessor(XML_Parser parser, const char *s, const char *end, #ifdef XML_DTD -static enum XML_Error PTRCALL +static enum XML_Error externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { enum XML_Error result = initializeEncoding(parser); @@ -4992,7 +5305,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, /* we know now that XML_Parse(Buffer) has been called, so we consider the external parameter entity read */ - parser->m_dtd->paramEntityRead = XML_TRUE; + parser->m_dtd->paramEntityRead = true; if (parser->m_prologState.inEntityValue) { parser->m_processor = entityValueInitProcessor; @@ -5003,7 +5316,7 @@ externalParEntInitProcessor(XML_Parser parser, const char *s, const char *end, } } -static enum XML_Error PTRCALL +static enum XML_Error entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { int tok; @@ -5087,7 +5400,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } } -static enum XML_Error PTRCALL +static enum XML_Error externalParEntProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { const char *next = s; @@ -5133,7 +5446,7 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, XML_ACCOUNT_DIRECT); } -static enum XML_Error PTRCALL +static enum XML_Error entityValueProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { const char *start = s; @@ -5180,7 +5493,7 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, #endif /* XML_DTD */ -static enum XML_Error PTRCALL +static enum XML_Error prologProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { const char *next = s; @@ -5426,7 +5739,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_useForeignDTD) entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; + dtd->paramEntityRead = false; beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, @@ -5476,7 +5789,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! entity) return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; - dtd->paramEntityRead = XML_FALSE; + dtd->paramEntityRead = false; beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, @@ -5819,7 +6132,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_FALSE; + parser->m_declEntity->is_param = false; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -5849,7 +6162,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } else { poolFinish(&dtd->pool); parser->m_declEntity->publicId = NULL; - parser->m_declEntity->is_param = XML_TRUE; + parser->m_declEntity->is_param = true; /* if we have a parent parser or are reading an internal parameter entity, then the entity declaration is not considered "internal" */ @@ -6084,8 +6397,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_RECURSIVE_ENTITY_REF; if (entity->textPtr) { enum XML_Error result; - XML_Bool betweenDecl - = (role == XML_ROLE_PARAM_ENTITY_REF ? XML_TRUE : XML_FALSE); + bool betweenDecl = (role == XML_ROLE_PARAM_ENTITY_REF); result = processEntity(parser, entity, betweenDecl, ENTITY_INTERNAL); if (result != XML_ERROR_NONE) return result; @@ -6093,8 +6405,8 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; } if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6103,11 +6415,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { dtd->keepProcessing = dtd->standalone; @@ -6137,7 +6449,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; dtd->scaffLevel = 0; dtd->scaffCount = 0; - dtd->in_eldecl = XML_TRUE; + dtd->in_eldecl = true; handleDefault = XML_FALSE; } break; @@ -6166,7 +6478,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, afterHandler(parser); handleDefault = XML_FALSE; } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; } break; @@ -6248,7 +6560,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_handlerArg, parser->m_declElementType->name, model); afterHandler(parser); } - dtd->in_eldecl = XML_FALSE; + dtd->in_eldecl = false; dtd->contentStringLen = 0; } } @@ -6317,7 +6629,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* not reached */ } -static enum XML_Error PTRCALL +static enum XML_Error epilogProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { parser->m_processor = epilogProcessor; @@ -6396,7 +6708,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, } static enum XML_Error -processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, +processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, enum EntityType type) { OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; @@ -6431,8 +6743,8 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, if (! openEntity) return XML_ERROR_NO_MEMORY; } - entity->open = XML_TRUE; - entity->hasMore = XML_TRUE; + entity->open = true; + entity->hasMore = true; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); #endif @@ -6454,7 +6766,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error internalEntityProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { UNUSED_P(s); @@ -6504,7 +6816,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // Entity is complete. We cannot close it here since we need to first // process its possible inner entities (which are added to the // m_openInternalEntities during doProlog or doContent calls above) - entity->hasMore = XML_FALSE; + entity->hasMore = false; if (! entity->is_param && (openEntity->startTagLevel != parser->m_tagLevel)) { return XML_ERROR_ASYNC_ENTITY; @@ -6522,7 +6834,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // to false. This means we can directly remove the head of // m_openInternalEntities assert(parser->m_openInternalEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ @@ -6536,7 +6848,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_NONE; } -static enum XML_Error PTRCALL +static enum XML_Error errorProcessor(XML_Parser parser, const char *s, const char *end, const char **nextPtr) { UNUSED_P(s); @@ -6577,16 +6889,17 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // Check if entity is complete, if not, mark down how much of it is // processed. A XML_SUSPENDED check here is not required as // appendAttributeValue will never suspend the parser. - if (textEnd != nextInEntity) { + if (nextInEntity < textEnd) { entity->processed = (int)(nextInEntity - (const char *)entity->textPtr); continue; } + assert(nextInEntity == textEnd); // Entity is complete. We cannot close it here since we need to first // process its possible inner entities (which are added to the // m_openAttributeEntities during appendAttributeValue) - entity->hasMore = XML_FALSE; + entity->hasMore = false; continue; } // End of entity processing, "if" block skips the rest @@ -6599,7 +6912,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // with hasMore set to false. This means we can directly remove the head // of m_openAttributeEntities assert(parser->m_openAttributeEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ @@ -6894,8 +7207,8 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } if (entity->systemId) { if (parser->m_externalEntityRefHandler) { - dtd->paramEntityRead = XML_FALSE; - entity->open = XML_TRUE; + dtd->paramEntityRead = false; + entity->open = true; entityTrackingOnOpen(parser, entity, __LINE__); beforeHandler(parser); const int status = parser->m_externalEntityRefHandler( @@ -6904,12 +7217,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, afterHandler(parser); if (! status) { entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } entityTrackingOnClose(parser, entity, __LINE__); - entity->open = XML_FALSE; + entity->open = false; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; } else @@ -7046,7 +7359,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // Entity is complete. We cannot close it here since we need to first // process its possible inner entities (which are added to the // m_openValueEntities during storeEntityValue) - entity->hasMore = XML_FALSE; + entity->hasMore = false; continue; } // End of entity processing, "if" block skips the rest @@ -7059,7 +7372,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // with hasMore set to false. This means we can directly remove the head // of m_openValueEntities assert(parser->m_openValueEntities == openEntity); - entity->open = XML_FALSE; + entity->open = false; parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ @@ -7108,7 +7421,7 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { #endif /* XML_GE == 0 */ -static void FASTCALL +static void normalizeLines(XML_Char *s) { XML_Char *p; for (;; s++) { @@ -7240,7 +7553,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttForName), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) @@ -7274,13 +7587,26 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->value = value; att->isCdata = isCdata; if (! isCdata) - attId->maybeTokenized = XML_TRUE; + attId->maybeTokenized = true; - NAMED *const nameAddedOrFound - = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); - if (! nameAddedOrFound) + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(type->defaultAttForName), attId->name, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) return 0; + assert(nameAndDefaultAttribute->name == attId->name); + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = type->nDefaultAtts; + nameAndDefaultAttribute->initialized = true; + } + type->nDefaultAtts += 1; return 1; } @@ -7346,7 +7672,7 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, else id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, name + 6, sizeof(PREFIX)); - id->xmlns = XML_TRUE; + id->xmlns = true; } else { int i; for (i = 0; name[i]; i++) { @@ -7481,7 +7807,7 @@ setContext(XML_Parser parser, const XML_Char *context) { e = (ENTITY *)lookup(parser, &dtd->generalEntities, poolStart(&parser->m_tempPool), 0); if (e) - e->open = XML_TRUE; + e->open = true; if (*s != XML_T('\0')) s++; context = s; @@ -7536,7 +7862,7 @@ setContext(XML_Parser parser, const XML_Char *context) { return XML_TRUE; } -static void FASTCALL +static void normalizePublicId(XML_Char *publicId) { XML_Char *p = publicId; XML_Char *s; @@ -7569,13 +7895,13 @@ dtdCreate(XML_Parser parser) { hashTableInit(&(p->attributeIds), parser); hashTableInit(&(p->prefixes), parser); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; + p->paramEntityRead = false; hashTableInit(&(p->paramEntities), parser); #endif /* XML_DTD */ p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; p->scaffIndex = NULL; p->scaffIndexSize = 0; p->scaffold = NULL; @@ -7598,12 +7924,12 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD - p->paramEntityRead = XML_FALSE; + p->paramEntityRead = false; hashTableClear(&(p->paramEntities)); #endif /* XML_DTD */ hashTableClear(&(p->elementTypes)); @@ -7614,7 +7940,7 @@ dtdReset(DTD *p, XML_Parser parser) { p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; - p->in_eldecl = XML_FALSE; + p->in_eldecl = false; FREE(parser, p->scaffIndex); p->scaffIndex = NULL; @@ -7640,7 +7966,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - hashTableDestroy(&(e->defaultAttsNames)); + hashTableDestroy(&(e->defaultAttForName)); FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); @@ -7733,8 +8059,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (! newE) return 0; - if (! newE->defaultAttsNames.parser) - hashTableInit(&(newE->defaultAttsNames), parser); + if (! newE->defaultAttForName.parser) + hashTableInit(&(newE->defaultAttForName), parser); if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. */ @@ -7767,11 +8093,22 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), - attributeName, sizeof(NAMED)); - if (! nameAddedOrFound) { + NAME_AND_DEFAULT_ATTRIBUTE *const nameAndDefaultAttribute + = (NAME_AND_DEFAULT_ATTRIBUTE *)lookup( + parser, &(newE->defaultAttForName), attributeName, + sizeof(NAME_AND_DEFAULT_ATTRIBUTE)); + if (! nameAndDefaultAttribute) { return 0; } + + // NOTE: The XML 1.0r4 spec says: + // "When more than one definition is provided for the same attribute of a + // given element type, the first declaration is binding and later + // declarations are ignored." + if (! nameAndDefaultAttribute->initialized) { + nameAndDefaultAttribute->attIndex = i; + nameAndDefaultAttribute->initialized = true; + } } } @@ -7868,23 +8205,27 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, #define INIT_POWER 6 -static XML_Bool FASTCALL -keyeq(KEY s1, KEY s2) { +// Compares two strings `s1` and `s2` whereas: +// - `s2` is zero-terminated but +// - `s1` is made up of exactly (not just up to) `s1len` non-zero characters. +XML_NONTESTING_STATIC XML_Bool +keyeq(KEY s1, size_t s1len, KEY s2) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T - return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (wcsncmp(s1, s2, s1len) == 0 && s2[s1len] == L'\0') ? XML_TRUE + : XML_FALSE; # else - for (; *s1 == *s2; s1++, s2++) - if (*s1 == 0) - return XML_TRUE; - return XML_FALSE; + for (; s1len > 0 && *s1 == *s2; s1len--, s1++, s2++) + ; /* no loop body! */ + return ((s1len == 0) && (*s2 == 0)) ? XML_TRUE : XML_FALSE; # endif #else - return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; + return (strncmp(s1, s2, s1len) == 0 && s2[s1len] == '\0') ? XML_TRUE + : XML_FALSE; #endif } -static size_t +XML_NONTESTING_STATIC size_t keylen(KEY s) { return xcslen(s); } @@ -7897,19 +8238,39 @@ copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { *key = rootParser->m_hash_secret_salt_128; } -static unsigned long FASTCALL -hash(XML_Parser parser, KEY s) { +static unsigned long +hash(XML_Parser parser, KEY s, size_t keyLen) { struct siphash state; struct sipkey key; (void)sip24_valid; copy_salt_to_sipkey(parser, &key); sip24_init(&state, &key); - sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + sip24_update(&state, s, keyLen * sizeof(XML_Char)); return (unsigned long)sip24_final(&state); } -static NAMED * -lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { +// Function `lookupWithLength` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +// NOTE: Read-only lookup does not need zero-terminated keys but +// read-write mode does, because keys can be re-hashed later and the +// hash table does not store key length information. +// +XML_NONTESTING_STATIC NAMED * +lookupWithLength(XML_Parser parser, HASH_TABLE *table, KEY name, size_t nameLen, + size_t createSize) { size_t i; if (table->size == 0) { size_t tsize; @@ -7925,14 +8286,14 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { return NULL; } memset(table->v, 0, tsize); - i = hash(parser, name) & ((unsigned long)table->size - 1); + i = hash(parser, name, nameLen) & ((unsigned long)table->size - 1); } else { - unsigned long h = hash(parser, name); + unsigned long h = hash(parser, name, nameLen); unsigned long mask = (unsigned long)table->size - 1; unsigned char step = 0; i = h & mask; while (table->v[i]) { - if (keyeq(name, table->v[i]->name)) + if (keyeq(name, nameLen, table->v[i]->name)) return table->v[i]; if (! step) step = PROBE_STEP(h, mask, table->power); @@ -7965,7 +8326,8 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { memset(newV, 0, tsize); for (i = 0; i < table->size; i++) if (table->v[i]) { - unsigned long newHash = hash(parser, table->v[i]->name); + KEY const key = table->v[i]->name; + unsigned long newHash = hash(parser, key, keylen(key)); size_t j = newHash & newMask; step = 0; while (newV[j]) { @@ -7988,16 +8350,37 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } + assert(createSize >= sizeof(NAMED)); table->v[i] = (NAMED*)MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); - table->v[i]->name = name; + table->v[i]->name = name; // NOTE: This requires and assumes zero termination! (table->used)++; return table->v[i]; } -static void FASTCALL +// Function `lookup` can be used to either… +// +// a) check whether an element with key `name` exists in the given hash table +// (read-only mode where `createSize == 0`) or +// +// b) check whether an element with key `name` exists in the given hash table +// *and* insert it if missing (i.e. read-write mode where `createSize != 0`. +// +// When inserting, a block of `createSize` number of bytes will be allocated +// and set to zero, and the resulting block of memory will be considered +// to start with a `NAMED` structure, and `->name = name;` is performed. +// The fact that all other bytes in the structure are initially zero can +// be used to tell cases "existed and found" and "newly inserted" apart +// with the structure returned. +// +XML_NONTESTING_STATIC NAMED * +lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { + return lookupWithLength(parser, table, name, keylen(name), createSize); +} + +XML_NONTESTING_STATIC void hashTableClear(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) { @@ -8007,7 +8390,7 @@ hashTableClear(HASH_TABLE *table) { table->used = 0; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableDestroy(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) @@ -8015,7 +8398,7 @@ hashTableDestroy(HASH_TABLE *table) { FREE(table->parser, table->v); } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->power = 0; p->size = 0; @@ -8024,13 +8407,13 @@ hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->parser = parser; } -static void FASTCALL +XML_NONTESTING_STATIC void hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; iter->end = iter->p ? iter->p + table->size : NULL; } -static NAMED *FASTCALL +XML_NONTESTING_STATIC NAMED * hashTableIterNext(HASH_TABLE_ITER *iter) { while (iter->p != iter->end) { NAMED *tem = *(iter->p)++; @@ -8040,7 +8423,7 @@ hashTableIterNext(HASH_TABLE_ITER *iter) { return NULL; } -static void FASTCALL +static void poolInit(STRING_POOL *pool, XML_Parser parser) { pool->blocks = NULL; pool->freeBlocks = NULL; @@ -8050,7 +8433,7 @@ poolInit(STRING_POOL *pool, XML_Parser parser) { pool->parser = parser; } -static void FASTCALL +static void poolClear(STRING_POOL *pool) { if (! pool->freeBlocks) pool->freeBlocks = pool->blocks; @@ -8069,7 +8452,7 @@ poolClear(STRING_POOL *pool) { pool->end = NULL; } -static void FASTCALL +static void poolDestroy(STRING_POOL *pool) { BLOCK *p = pool->blocks; while (p) { @@ -8102,7 +8485,7 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, return pool->start; } -static const XML_Char *FASTCALL +static const XML_Char * poolCopyString(STRING_POOL *pool, const XML_Char *s) { if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) return NULL; @@ -8113,7 +8496,7 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) { // A version of `poolCopyString` that does not call `poolFinish` // and reverts any partial advancement upon failure. -static const XML_Char *FASTCALL +static const XML_Char * poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { const XML_Char *const original = s; do { @@ -8151,7 +8534,7 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { return s; } -static const XML_Char *FASTCALL +static const XML_Char * poolAppendString(STRING_POOL *pool, const XML_Char *s) { if (! poolAppendChars(pool, s, xcslen(s))) return NULL; @@ -8196,7 +8579,7 @@ poolBytesToAllocateFor(int blockSize) { } } -static XML_Bool FASTCALL +static XML_Bool poolGrow(STRING_POOL *pool) { if (pool->freeBlocks) { if (pool->start == NULL) { @@ -8215,7 +8598,7 @@ poolGrow(STRING_POOL *pool) { pool->freeBlocks = tem; memcpy(pool->blocks->s, pool->start, (pool->end - pool->start) * sizeof(XML_Char)); - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = pool->blocks->s; pool->end = pool->start + pool->blocks->size; return XML_TRUE; @@ -8228,7 +8611,8 @@ poolGrow(STRING_POOL *pool) { /* NOTE: Needs to be calculated prior to calling `realloc` to avoid dangling pointers: */ - const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + const ptrdiff_t offsetInsideBlock + = EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); if (blockSize < 0) { /* This condition traps a situation where either more than @@ -8291,15 +8675,16 @@ poolGrow(STRING_POOL *pool) { tem->next = pool->blocks; pool->blocks = tem; if (pool->ptr != pool->start) - memcpy(tem->s, pool->start, (pool->ptr - pool->start) * sizeof(XML_Char)); - pool->ptr = tem->s + (pool->ptr - pool->start); + memcpy(tem->s, pool->start, + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start) * sizeof(XML_Char)); + pool->ptr = tem->s + EXPAT_SAFE_PTR_DIFF(pool->ptr, pool->start); pool->start = tem->s; pool->end = tem->s + blockSize; } return XML_TRUE; } -static bool FASTCALL +static bool poolGrowUntil(STRING_POOL *pool, size_t needed) { for (;;) { const size_t available = pool->end - pool->ptr; @@ -8312,7 +8697,7 @@ poolGrowUntil(STRING_POOL *pool, size_t needed) { } } -static int FASTCALL +static int nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ CONTENT_SCAFFOLD *me; @@ -8534,8 +8919,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; - if (! ret->defaultAttsNames.parser) - hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); + if (! ret->defaultAttForName.parser) + hashTableInit(&(ret->defaultAttForName), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/base/poco/XML/src/xmlrole.c b/base/poco/XML/src/xmlrole.c index 2c6133f236a9f..15aff759b45d3 100644 --- a/base/poco/XML/src/xmlrole.c +++ b/base/poco/XML/src/xmlrole.c @@ -12,7 +12,7 @@ Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2002-2003 Fred L. Drake, Jr. Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2023 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na @@ -36,6 +36,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -118,9 +120,8 @@ static const char KW_SYSTEM[] # define setTopLevel(state) ((state)->handler = internalSubset) #endif /* not XML_DTD */ -typedef int PTRCALL PROLOG_HANDLER(PROLOG_STATE *state, int tok, - const char *ptr, const char *end, - const ENCODING *enc); +typedef int PROLOG_HANDLER(PROLOG_STATE *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, doctype3, doctype4, doctype5, internalSubset, entity0, entity1, entity2, @@ -134,9 +135,9 @@ static PROLOG_HANDLER prolog0, prolog1, prolog2, doctype0, doctype1, doctype2, #endif /* XML_DTD */ declClose, error; -static int FASTCALL common(PROLOG_STATE *state, int tok); +static int common(PROLOG_STATE *state, int tok); -static int PTRCALL +static int prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -167,7 +168,7 @@ prolog0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -199,7 +200,7 @@ prolog1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -219,7 +220,7 @@ prolog2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -236,7 +237,7 @@ doctype0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -262,7 +263,7 @@ doctype1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -278,7 +279,7 @@ doctype2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -294,7 +295,7 @@ doctype3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -313,7 +314,7 @@ doctype4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -329,7 +330,7 @@ doctype5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -374,7 +375,7 @@ internalSubset(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { state->handler = externalSubset1; @@ -383,7 +384,7 @@ externalSubset0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return externalSubset1(state, tok, ptr, end, enc); } -static int PTRCALL +static int externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -411,7 +412,7 @@ externalSubset1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -430,7 +431,7 @@ entity0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -446,7 +447,7 @@ entity1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -470,7 +471,7 @@ entity2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -486,7 +487,7 @@ entity3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -502,7 +503,7 @@ entity4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -521,7 +522,7 @@ entity5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -538,7 +539,7 @@ entity6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -562,7 +563,7 @@ entity7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -578,7 +579,7 @@ entity8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -594,7 +595,7 @@ entity9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -610,7 +611,7 @@ entity10(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -626,7 +627,7 @@ notation0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -646,7 +647,7 @@ notation1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -662,7 +663,7 @@ notation2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -679,7 +680,7 @@ notation3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -699,7 +700,7 @@ notation4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -716,7 +717,7 @@ attlist0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -736,7 +737,7 @@ attlist1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -766,7 +767,7 @@ attlist2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -784,7 +785,7 @@ attlist3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -803,7 +804,7 @@ attlist4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -819,7 +820,7 @@ attlist5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -835,7 +836,7 @@ attlist6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -855,7 +856,7 @@ attlist7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* default value */ -static int PTRCALL +static int attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -885,7 +886,7 @@ attlist8(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -901,7 +902,7 @@ attlist9(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -918,7 +919,7 @@ element0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -944,7 +945,7 @@ element1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -978,7 +979,7 @@ element2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1002,7 +1003,7 @@ element3(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1019,7 +1020,7 @@ element4(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1039,7 +1040,7 @@ element5(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1068,7 +1069,7 @@ element6(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1117,7 +1118,7 @@ element7(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #ifdef XML_DTD -static int PTRCALL +static int condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { switch (tok) { @@ -1137,7 +1138,7 @@ condSect0(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1154,7 +1155,7 @@ condSect1(PROLOG_STATE *state, int tok, const char *ptr, const char *end, return common(state, tok); } -static int PTRCALL +static int condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1172,7 +1173,7 @@ condSect2(PROLOG_STATE *state, int tok, const char *ptr, const char *end, #endif /* XML_DTD */ -static int PTRCALL +static int declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(ptr); @@ -1208,7 +1209,7 @@ declClose(PROLOG_STATE *state, int tok, const char *ptr, const char *end, * * LCOV_EXCL_START */ -static int PTRCALL +static int error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, const ENCODING *enc) { UNUSED_P(state); @@ -1220,7 +1221,7 @@ error(PROLOG_STATE *state, int tok, const char *ptr, const char *end, } /* LCOV_EXCL_STOP */ -static int FASTCALL +static int common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) diff --git a/base/poco/XML/src/xmlrole.h b/base/poco/XML/src/xmlrole.h index 4cb28ddf9ca4f..9bb732df7773f 100644 --- a/base/poco/XML/src/xmlrole.h +++ b/base/poco/XML/src/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Karl Waclawek Copyright (c) 2002 Fred L. Drake, Jr. - Copyright (c) 2017-2025 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -31,6 +31,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlRole_INCLUDED @@ -110,8 +112,8 @@ enum { }; typedef struct prolog_state { - int(PTRCALL *handler)(struct prolog_state *state, int tok, const char *ptr, - const char *end, const ENCODING *enc); + int (*handler)(struct prolog_state *state, int tok, const char *ptr, + const char *end, const ENCODING *enc); unsigned level; int role_none; # ifdef XML_DTD diff --git a/base/poco/XML/src/xmltok.c b/base/poco/XML/src/xmltok.c index fe8a780b625d3..c36dde216214e 100644 --- a/base/poco/XML/src/xmltok.c +++ b/base/poco/XML/src/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Pascal Cuoq Copyright (c) 2016 Don Lewis Copyright (c) 2017 Rhodri James @@ -26,6 +26,8 @@ Copyright (c) 2023 Hanno Böck Copyright (c) 2025 Alfonso Gregory Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Afonso Januário Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -46,6 +48,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #include "expat_config.h" @@ -134,20 +138,20 @@ : ((p)[1] & 0x80) == 0 \ || ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0))) -static int PTRFASTCALL +static int isNever(const ENCODING *enc, const char *p) { UNUSED_P(enc); UNUSED_P(p); return 0; } -static int PTRFASTCALL +static int utf8_isName2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(namePages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isName3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(namePages, (const unsigned char *)p); @@ -155,13 +159,13 @@ utf8_isName3(const ENCODING *enc, const char *p) { #define utf8_isName4 isNever -static int PTRFASTCALL +static int utf8_isNmstrt2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isNmstrt3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p); @@ -169,19 +173,19 @@ utf8_isNmstrt3(const ENCODING *enc, const char *p) { #define utf8_isNmstrt4 isNever -static int PTRFASTCALL +static int utf8_isInvalid2(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID2((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid3(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID3((const unsigned char *)p); } -static int PTRFASTCALL +static int utf8_isInvalid4(const ENCODING *enc, const char *p) { UNUSED_P(enc); return UTF8_INVALID4((const unsigned char *)p); @@ -190,36 +194,20 @@ utf8_isInvalid4(const ENCODING *enc, const char *p) { struct normal_encoding { ENCODING enc; unsigned char type[256]; -#ifdef XML_MIN_SIZE - int(PTRFASTCALL *byteType)(const ENCODING *, const char *); - int(PTRFASTCALL *isNameMin)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *); - int(PTRFASTCALL *byteToAscii)(const ENCODING *, const char *); - int(PTRCALL *charMatches)(const ENCODING *, const char *, int); -#endif /* XML_MIN_SIZE */ - int(PTRFASTCALL *isName2)(const ENCODING *, const char *); - int(PTRFASTCALL *isName3)(const ENCODING *, const char *); - int(PTRFASTCALL *isName4)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *); - int(PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid2)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid3)(const ENCODING *, const char *); - int(PTRFASTCALL *isInvalid4)(const ENCODING *, const char *); + int (*isName2)(const ENCODING *, const char *); + int (*isName3)(const ENCODING *, const char *); + int (*isName4)(const ENCODING *, const char *); + int (*isNmstrt2)(const ENCODING *, const char *); + int (*isNmstrt3)(const ENCODING *, const char *); + int (*isNmstrt4)(const ENCODING *, const char *); + int (*isInvalid2)(const ENCODING *, const char *); + int (*isInvalid3)(const ENCODING *, const char *); + int (*isInvalid4)(const ENCODING *, const char *); }; #define AS_NORMAL_ENCODING(enc) ((const struct normal_encoding *)(enc)) -#ifdef XML_MIN_SIZE - -# define STANDARD_VTABLE(E) \ - E##byteType, E##isNameMin, E##isNmstrtMin, E##byteToAscii, E##charMatches, - -#else - -# define STANDARD_VTABLE(E) /* as nothing */ - -#endif +#define STANDARD_VTABLE(E) /* as nothing */ #define NORMAL_VTABLE(E) \ E##isName2, E##isName3, E##isName4, E##isNmstrt2, E##isNmstrt3, \ @@ -230,80 +218,33 @@ struct normal_encoding { /* isNmstrt2 */ NULL, /* isNmstrt3 */ NULL, /* isNmstrt4 */ NULL, \ /* isInvalid2 */ NULL, /* isInvalid3 */ NULL, /* isInvalid4 */ NULL -static int FASTCALL checkCharRefNumber(int result); +#define UTF16_NULL_VTABLE(E) NULL_VTABLE + +static int checkCharRefNumber(int result); #include "xmltok_impl.h" #include "ascii.h" -#ifdef XML_MIN_SIZE -# define sb_isNameMin isNever -# define sb_isNmstrtMin isNever -#endif - -#ifdef XML_MIN_SIZE -# define MINBPC(enc) ((enc)->minBytesPerChar) -#else /* minimum bytes per character */ -# define MINBPC(enc) 1 -#endif +#define MINBPC(enc) 1 #define SB_BYTE_TYPE(enc, p) \ (((const struct normal_encoding *)(enc))->type[(unsigned char)*(p)]) -#ifdef XML_MIN_SIZE -static int PTRFASTCALL -sb_byteType(const ENCODING *enc, const char *p) { - return SB_BYTE_TYPE(enc, p); -} -# define BYTE_TYPE(enc, p) (AS_NORMAL_ENCODING(enc)->byteType(enc, p)) -#else -# define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#endif +#define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p) -#ifdef XML_MIN_SIZE -# define BYTE_TO_ASCII(enc, p) (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p)) -static int PTRFASTCALL -sb_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return *p; -} -#else -# define BYTE_TO_ASCII(enc, p) (*(p)) -#endif +#define BYTE_TO_ASCII(enc, p) (*(p)) #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n \ - && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#else -# define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#endif +#define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) -#ifdef XML_MIN_SIZE -# define IS_NAME_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p)) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) \ - (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p)) -#else -# define IS_NAME_CHAR_MINBPC(enc, p) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#endif +#define IS_NAME_CHAR_MINBPC(enc, p) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) (0) -#ifdef XML_MIN_SIZE -# define CHAR_MATCHES(enc, p, c) \ - (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c)) -static int PTRCALL -sb_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return *p == c; -} -#else /* c is an ASCII character */ -# define CHAR_MATCHES(enc, p, c) (*(p) == (c)) -#endif +#define CHAR_MATCHES(enc, p, c) (*(p) == (c)) #define PREFIX(ident) normal_##ident #define XML_TOK_IMPL_C @@ -366,7 +307,7 @@ _INTERNAL_trim_to_complete_utf8_characters(const char *from, *fromLimRef = fromLim; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { bool input_incomplete = false; @@ -405,7 +346,7 @@ utf8_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, return XML_CONVERT_COMPLETED; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result utf8_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { enum XML_Convert_Result res = XML_CONVERT_COMPLETED; @@ -503,7 +444,7 @@ static const struct normal_encoding internal_utf8_encoding }, STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -526,7 +467,7 @@ latin1_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result latin1_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { UNUSED_P(enc); @@ -561,7 +502,7 @@ static const struct normal_encoding latin1_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result ascii_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { UNUSED_P(enc); @@ -596,7 +537,7 @@ static const struct normal_encoding ascii_encoding }, STANDARD_VTABLE(sb_) NULL_VTABLE}; -static int PTRFASTCALL +static int unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ @@ -623,7 +564,7 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF8(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf8( \ + static enum XML_Convert_Result E##toUtf8( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ char **toP, const char *toLim) { \ const char *from = *fromP; \ @@ -700,15 +641,16 @@ unicode_byte_type(char hi, char lo) { } #define DEFINE_UTF16_TO_UTF16(E) \ - static enum XML_Convert_Result PTRCALL E##toUtf16( \ + static enum XML_Convert_Result E##toUtf16( \ const ENCODING *enc, const char **fromP, const char *fromLim, \ unsigned short **toP, const unsigned short *toLim) { \ enum XML_Convert_Result res = XML_CONVERT_COMPLETED; \ UNUSED_P(enc); \ fromLim = *fromP + (((fromLim - *fromP) >> 1) << 1); /* shrink to even */ \ - /* Avoid copying first half only of surrogate */ \ + /* Avoid copying the first half (2 bytes) of surrogate pairs (4 bytes) */ \ if (fromLim - *fromP > ((toLim - *toP) << 1) \ - && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) { \ + && /* are the last two bytes a high surrogate (0xD800-0xDBFF)? */ \ + (GET_HI(fromLim - 2) & 0xFC) == 0xD8) { \ fromLim -= 2; \ res = XML_CONVERT_INPUT_INCOMPLETE; \ } \ @@ -746,70 +688,38 @@ DEFINE_UTF16_TO_UTF16(big2_) UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0]) #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define LITTLE2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[3] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -little2_byteType(const ENCODING *enc, const char *p) { - return LITTLE2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -little2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -little2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return LITTLE2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -little2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -little2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return LITTLE2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16 +#undef PREFIX +#define PREFIX(ident) little2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) LITTLE2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) little2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -825,7 +735,7 @@ static const struct normal_encoding little2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -843,7 +753,7 @@ static const struct normal_encoding little2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #if BYTEORDER != 4321 @@ -855,7 +765,7 @@ static const struct normal_encoding internal_little2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; # endif @@ -867,7 +777,7 @@ static const struct normal_encoding internal_little2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(little2_) NULL_VTABLE}; + STANDARD_VTABLE(little2_) UTF16_NULL_VTABLE(little2_)}; #endif @@ -879,70 +789,38 @@ static const struct normal_encoding internal_little2_encoding UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1]) #define BIG2_IS_NMSTRT_CHAR_MINBPC(p) \ UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1]) +/* A 4-byte UTF-16 character is a surrogate pair; byteType only reports BT_LEAD4 + for a high surrogate, so the pair is invalid unless the second unit is a low + surrogate (U+DC00..U+DFFF, i.e. high byte 0xDC..0xDF). */ +#define BIG2_IS_INVALID_CHAR(p, n) \ + ((n) == 4 && ((unsigned char)(p)[2] & 0xFC) != 0xDC) -#ifdef XML_MIN_SIZE - -static int PTRFASTCALL -big2_byteType(const ENCODING *enc, const char *p) { - return BIG2_BYTE_TYPE(enc, p); -} - -static int PTRFASTCALL -big2_byteToAscii(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_BYTE_TO_ASCII(p); -} - -static int PTRCALL -big2_charMatches(const ENCODING *enc, const char *p, int c) { - UNUSED_P(enc); - return BIG2_CHAR_MATCHES(p, c); -} - -static int PTRFASTCALL -big2_isNameMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NAME_CHAR_MINBPC(p); -} - -static int PTRFASTCALL -big2_isNmstrtMin(const ENCODING *enc, const char *p) { - UNUSED_P(enc); - return BIG2_IS_NMSTRT_CHAR_MINBPC(p); -} - -# undef VTABLE -# define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16 +#undef PREFIX +#define PREFIX(ident) big2_##ident +#define MINBPC(enc) 2 +/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ +#define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) +#define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) +#define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) +#define IS_NAME_CHAR(enc, p, n) 0 +#define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) +#define IS_NMSTRT_CHAR(enc, p, n) (0) +#define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) +#define IS_INVALID_CHAR(enc, p, n) BIG2_IS_INVALID_CHAR(p, n) -#else /* not XML_MIN_SIZE */ +#define XML_TOK_IMPL_C +#include "xmltok_impl.c" +#undef XML_TOK_IMPL_C -# undef PREFIX -# define PREFIX(ident) big2_##ident -# define MINBPC(enc) 2 -/* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */ -# define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p) -# define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(p) -# define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(p, c) -# define IS_NAME_CHAR(enc, p, n) 0 -# define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(p) -# define IS_NMSTRT_CHAR(enc, p, n) (0) -# define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(p) - -# define XML_TOK_IMPL_C -# include "xmltok_impl.c" -# undef XML_TOK_IMPL_C - -# undef MINBPC -# undef BYTE_TYPE -# undef BYTE_TO_ASCII -# undef CHAR_MATCHES -# undef IS_NAME_CHAR -# undef IS_NAME_CHAR_MINBPC -# undef IS_NMSTRT_CHAR -# undef IS_NMSTRT_CHAR_MINBPC -# undef IS_INVALID_CHAR - -#endif /* not XML_MIN_SIZE */ +#undef MINBPC +#undef BYTE_TYPE +#undef BYTE_TO_ASCII +#undef CHAR_MATCHES +#undef IS_NAME_CHAR +#undef IS_NAME_CHAR_MINBPC +#undef IS_NMSTRT_CHAR +#undef IS_NMSTRT_CHAR_MINBPC +#undef IS_INVALID_CHAR #ifdef XML_NS @@ -958,7 +836,7 @@ static const struct normal_encoding big2_encoding_ns # include "asciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif @@ -976,7 +854,7 @@ static const struct normal_encoding big2_encoding #undef BT_COLON #include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #if BYTEORDER != 1234 @@ -988,7 +866,7 @@ static const struct normal_encoding internal_big2_encoding_ns # include "iasciitab.h" # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; # endif @@ -1000,13 +878,13 @@ static const struct normal_encoding internal_big2_encoding # undef BT_COLON # include "latin1tab.h" }, - STANDARD_VTABLE(big2_) NULL_VTABLE}; + STANDARD_VTABLE(big2_) UTF16_NULL_VTABLE(big2_)}; #endif #undef PREFIX -static int FASTCALL +static int streqci(const char *s1, const char *s2) { for (;;) { char c1 = *s1++; @@ -1027,7 +905,7 @@ streqci(const char *s1, const char *s2) { return 1; } -static void PTRCALL +static void initUpdatePosition(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { UNUSED_P(enc); @@ -1045,7 +923,7 @@ toAscii(const ENCODING *enc, const char *ptr, const char *end) { return buf[0]; } -static int FASTCALL +static int isSpace(int c) { switch (c) { case 0x20: @@ -1151,6 +1029,34 @@ static const char KW_yes[] = {ASCII_y, ASCII_e, ASCII_s, '\0'}; static const char KW_no[] = {ASCII_n, ASCII_o, '\0'}; +static const char KW_1_dot[] = {ASCII_1, ASCII_PERIOD, '\0'}; + +/* Checks a version pseudo-attribute value against the VersionNum production. + XML 1.0 Fourth Edition only allows the literal "1.0", but the Fifth + Edition relaxed this to "1." followed by one or more digits, since Expat + only implements 1.0 itself but plans to track the Fifth Edition's laxer + grammar here so that "1.1" and similar aren't rejected only to have that + rejection reverted later. Returns true for a value matching "1.[0-9]+". + val/valEnd bound the value itself; valEnd is the upper bound used when + decoding the individual characters between them. */ +static bool +checkXmlDeclVersionNum(const ENCODING *enc, const char *val, + const char *valEnd) { + if (valEnd - val < 2 * enc->minBytesPerChar + || ! XmlNameMatchesAscii(enc, val, val + 2 * enc->minBytesPerChar, + KW_1_dot)) + return false; + val += 2 * enc->minBytesPerChar; + if (val == valEnd) + return false; + for (; val != valEnd; val += enc->minBytesPerChar) { + int c = toAscii(enc, val, valEnd); + if (c < ASCII_0 || c > ASCII_9) + return false; + } + return true; +} + static int doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, const char *), @@ -1178,6 +1084,22 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, *versionPtr = val; if (versionEndPtr) *versionEndPtr = ptr; + /* The version number must not be empty; VersionNum requires at least + one character. The encoding and standalone pseudo-attributes below + already reject an empty value, so keep version consistent. */ + if (val == ptr - enc->minBytesPerChar) { + *badPtr = val; + return 0; + } + /* Expat implements XML 1.0 only, so any version outside the "1.0"/"1.x" + family is rejected. Following the Fifth Edition's VersionNum + production (rather than the Fourth Edition's exact "1.0") avoids + rejecting "1.1" now only to have to revert that once Expat tracks + the newer edition. */ + if (! checkXmlDeclVersionNum(enc, val, ptr - enc->minBytesPerChar)) { + *badPtr = val; + return 0; + } if (! parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) { *badPtr = ptr; return 0; @@ -1232,7 +1154,7 @@ doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *, const char *, return 1; } -static int FASTCALL +static int checkCharRefNumber(int result) { switch (result >> 8) { case 0xD8: @@ -1256,7 +1178,7 @@ checkCharRefNumber(int result) { return result; } -int FASTCALL +int XmlUtf8Encode(int c, char *buf) { enum { /* minN is minimum legal resulting value for N byte sequence */ @@ -1292,7 +1214,7 @@ XmlUtf8Encode(int c, char *buf) { return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } -int FASTCALL +int XmlUtf16Encode(int charNum, unsigned short *buf) { if (charNum < 0) return 0; @@ -1324,7 +1246,7 @@ XmlSizeOfUnknownEncoding(void) { return sizeof(struct unknown_encoding); } -static int PTRFASTCALL +static int unknown_isName(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1333,7 +1255,7 @@ unknown_isName(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isNmstrt(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); @@ -1342,14 +1264,14 @@ unknown_isNmstrt(const ENCODING *enc, const char *p) { return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF); } -static int PTRFASTCALL +static int unknown_isInvalid(const ENCODING *enc, const char *p) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); int c = uenc->convert(uenc->userData, p); return (c & ~0xFFFF) || checkCharRefNumber(c) < 0; } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, char **toP, const char *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1379,7 +1301,7 @@ unknown_toUtf8(const ENCODING *enc, const char **fromP, const char *fromLim, } } -static enum XML_Convert_Result PTRCALL +static enum XML_Convert_Result unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, unsigned short **toP, const unsigned short *toLim) { const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc); @@ -1503,7 +1425,7 @@ static const char KW_UTF_16LE[] = {ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E, '\0'}; -static int FASTCALL +static int getEncodingIndex(const char *name) { static const char *const encodingNames[] = { KW_ISO_8859_1, KW_US_ASCII, KW_UTF_8, KW_UTF_16, KW_UTF_16BE, KW_UTF_16LE, diff --git a/base/poco/XML/src/xmltok.h b/base/poco/XML/src/xmltok.h index f949ebf9c1714..4f6fb3850cafc 100644 --- a/base/poco/XML/src/xmltok.h +++ b/base/poco/XML/src/xmltok.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2005 Karl Waclawek - Copyright (c) 2016-2024 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Licensed under the MIT license: @@ -32,11 +32,15 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifndef XmlTok_INCLUDED # define XmlTok_INCLUDED 1 +# include // uint64_t + # ifdef __cplusplus extern "C" { # endif @@ -145,8 +149,8 @@ extern "C" { typedef struct position { /* first line and first column are 0 not 1 */ - XML_Size lineNumber; - XML_Size columnNumber; + uint64_t lineNumber; + uint64_t columnNumber; } POSITION; typedef struct { @@ -159,41 +163,40 @@ typedef struct { struct encoding; typedef struct encoding ENCODING; -typedef int(PTRCALL *SCANNER)(const ENCODING *, const char *, const char *, - const char **); +typedef int (*SCANNER)(const ENCODING *, const char *, const char *, + const char **); enum XML_Convert_Result { XML_CONVERT_COMPLETED = 0, XML_CONVERT_INPUT_INCOMPLETE = 1, - XML_CONVERT_OUTPUT_EXHAUSTED - = 2 /* and therefore potentially input remaining as well */ + XML_CONVERT_OUTPUT_EXHAUSTED = 2 /* and therefore potentially input remaining + as well */ }; struct encoding { SCANNER scanners[XML_N_STATES]; SCANNER literalScanners[XML_N_LITERAL_TYPES]; - int(PTRCALL *nameMatchesAscii)(const ENCODING *, const char *, const char *, - const char *); - int(PTRFASTCALL *nameLength)(const ENCODING *, const char *); - const char *(PTRFASTCALL *skipS)(const ENCODING *, const char *); - int(PTRCALL *getAtts)(const ENCODING *enc, const char *ptr, int attsMax, - ATTRIBUTE *atts); - int(PTRFASTCALL *charRefNumber)(const ENCODING *enc, const char *ptr); - int(PTRCALL *predefinedEntityName)(const ENCODING *, const char *, - const char *); - void(PTRCALL *updatePosition)(const ENCODING *, const char *ptr, - const char *end, POSITION *); - int(PTRCALL *isPublicId)(const ENCODING *enc, const char *ptr, - const char *end, const char **badPtr); - enum XML_Convert_Result(PTRCALL *utf8Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, char **toP, - const char *toLim); - enum XML_Convert_Result(PTRCALL *utf16Convert)(const ENCODING *enc, - const char **fromP, - const char *fromLim, - unsigned short **toP, - const unsigned short *toLim); + int (*nameMatchesAscii)(const ENCODING *, const char *, const char *, + const char *); + int (*nameLength)(const ENCODING *, const char *); + const char *(*skipS)(const ENCODING *, const char *); + int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax, + ATTRIBUTE *atts); + int (*charRefNumber)(const ENCODING *enc, const char *ptr); + int (*predefinedEntityName)(const ENCODING *, const char *, const char *); + void (*updatePosition)(const ENCODING *, const char *ptr, const char *end, + POSITION *); + int (*isPublicId)(const ENCODING *enc, const char *ptr, const char *end, + const char **badPtr); + enum XML_Convert_Result (*utf8Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, char **toP, + const char *toLim); + enum XML_Convert_Result (*utf16Convert)(const ENCODING *enc, + const char **fromP, + const char *fromLim, + unsigned short **toP, + const unsigned short *toLim); int minBytesPerChar; char isUtf8; char isUtf16; @@ -293,8 +296,8 @@ int XmlInitEncoding(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncoding(void); const ENCODING *XmlGetUtf16InternalEncoding(void); -int FASTCALL XmlUtf8Encode(int charNumber, char *buf); -int FASTCALL XmlUtf16Encode(int charNumber, unsigned short *buf); +int XmlUtf8Encode(int charNumber, char *buf); +int XmlUtf16Encode(int charNumber, unsigned short *buf); int XmlSizeOfUnknownEncoding(void); typedef int(XMLCALL *CONVERTER)(void *userData, const char *p); @@ -314,6 +317,7 @@ const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert, void *userData); + # ifdef __cplusplus } # endif diff --git a/base/poco/XML/src/xmltok_impl.c b/base/poco/XML/src/xmltok_impl.c index d41b8e9e1a0c2..571af3698af7a 100644 --- a/base/poco/XML/src/xmltok_impl.c +++ b/base/poco/XML/src/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! +/* This file is included (from xmltok.c, 3 times)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2022 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Benjamin Peterson Copyright (c) 2018 Anton Maklakov @@ -18,6 +18,7 @@ Copyright (c) 2020 Boris Kolpackov Copyright (c) 2022 Martin Ettl Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -38,14 +39,12 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined -# define IS_INVALID_CHAR(enc, ptr, n) (0) -# endif - # define INVALID_LEAD_CASE(n, ptr, nextTokPtr) \ case BT_LEAD##n: \ if (end - ptr < n) \ @@ -142,7 +141,7 @@ /* ptr points to character following "= end) @@ -428,7 +427,7 @@ PREFIX(cdataSectionTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "= end) @@ -920,7 +919,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, /* ptr points to character following "%" */ -static int PTRCALL +static int PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -950,7 +949,7 @@ PREFIX(scanPercent)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { REQUIRE_CHAR(enc, ptr, end); @@ -980,7 +979,7 @@ PREFIX(scanPoundName)(const ENCODING *enc, const char *ptr, const char *end, return -XML_TOK_POUND_NAME; } -static int PTRCALL +static int PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { while (HAS_CHAR(enc, ptr, end)) { @@ -1014,7 +1013,7 @@ PREFIX(scanLit)(int open, const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_PARTIAL; } -static int PTRCALL +static int PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int tok; @@ -1258,7 +1257,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, return -tok; } -static int PTRCALL +static int PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1327,7 +1326,7 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_DATA_CHARS; } -static int PTRCALL +static int PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { const char *start; @@ -1394,7 +1393,7 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *end, # ifdef XML_DTD -static int PTRCALL +static int PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { int level = 0; @@ -1446,7 +1445,7 @@ PREFIX(ignoreSectionTok)(const ENCODING *enc, const char *ptr, const char *end, # endif /* XML_DTD */ -static int PTRCALL +static int PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, const char **badPtr) { ptr += MINBPC(enc); @@ -1506,7 +1505,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, first attsMax attributes are stored in atts. */ -static int PTRCALL +static int PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, ATTRIBUTE *atts) { enum { other, inName, inValue } state = inName; @@ -1599,7 +1598,7 @@ PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, /* not reached */ } -static int PTRFASTCALL +static int PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { int result = 0; /* skip &# */ @@ -1657,7 +1656,7 @@ PREFIX(charRefNumber)(const ENCODING *enc, const char *ptr) { return checkCharRefNumber(result); } -static int PTRCALL +static int PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, const char *end) { UNUSED_P(enc); @@ -1711,7 +1710,7 @@ PREFIX(predefinedEntityName)(const ENCODING *enc, const char *ptr, return 0; } -static int PTRCALL +static int PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, const char *end1, const char *ptr2) { UNUSED_P(enc); @@ -1730,7 +1729,7 @@ PREFIX(nameMatchesAscii)(const ENCODING *enc, const char *ptr1, return ptr1 == end1; } -static int PTRFASTCALL +static int PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { const char *start = ptr; for (;;) { @@ -1760,7 +1759,7 @@ PREFIX(nameLength)(const ENCODING *enc, const char *ptr) { } } -static const char *PTRFASTCALL +static const char * PREFIX(skipS)(const ENCODING *enc, const char *ptr) { for (;;) { switch (BYTE_TYPE(enc, ptr)) { @@ -1775,7 +1774,7 @@ PREFIX(skipS)(const ENCODING *enc, const char *ptr) { } } -static void PTRCALL +static void PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, POSITION *pos) { while (HAS_CHAR(enc, ptr, end)) { diff --git a/base/poco/XML/src/xmltok_impl.h b/base/poco/XML/src/xmltok_impl.h index eb18b31625d34..00fd893c060ab 100644 --- a/base/poco/XML/src/xmltok_impl.h +++ b/base/poco/XML/src/xmltok_impl.h @@ -29,6 +29,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ enum { diff --git a/base/poco/XML/src/xmltok_ns.c b/base/poco/XML/src/xmltok_ns.c index 0a23d7e8c4980..9601f53b68d8a 100644 --- a/base/poco/XML/src/xmltok_ns.c +++ b/base/poco/XML/src/xmltok_ns.c @@ -11,7 +11,8 @@ Copyright (c) 2002 Greg Stein Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek - Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,6 +33,8 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ #ifdef XML_TOK_NS_C @@ -61,14 +64,14 @@ static const ENCODING *const NS(encodings)[] = { &ns(utf8_encoding).enc /* NO_ENC */ }; -static int PTRCALL +static int NS(initScanProlog)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_PROLOG_STATE, ptr, end, nextTokPtr); } -static int PTRCALL +static int NS(initScanContent)(const ENCODING *enc, const char *ptr, const char *end, const char **nextTokPtr) { return initScan(NS(encodings), (const INIT_ENCODING *)enc, XML_CONTENT_STATE, diff --git a/base/poco/XML/src/xcsinc.c b/base/poco/XML/src/xmlxcs.c similarity index 65% rename from base/poco/XML/src/xcsinc.c rename to base/poco/XML/src/xmlxcs.c index 3597c2480bc96..ae5e042d778ff 100644 --- a/base/poco/XML/src/xcsinc.c +++ b/base/poco/XML/src/xmlxcs.c @@ -1,4 +1,4 @@ -/* This file is included from other .c files! +/* __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -6,7 +6,7 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 2022 Sebastian Pipping + Copyright (c) 2022-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -27,9 +27,21 @@ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT */ -static size_t +#include "xcs.h" + +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) +# include // for wcscmp, wcslen, wcsncmp +# endif +#else +# include // for strcmp, strlen, strncmp +#endif + +size_t xcslen(const XML_Char *s) { #ifdef XML_UNICODE # ifdef XML_UNICODE_WCHAR_T @@ -46,3 +58,33 @@ xcslen(const XML_Char *s) { return strlen(s); #endif } + +int +xcscmp(const XML_Char *a, const XML_Char *b) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcscmp(a, b); +# else + for (; a[0] && b[0] && a[0] == b[0]; a++, b++) + ; + return a[0] - b[0]; +# endif +#else + return strcmp(a, b); +#endif +} + +int +xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { +#if defined(XML_UNICODE) +# if defined(XML_UNICODE_WCHAR_T) + return wcsncmp(a, b, len); +# else + for (; len > 0 && a[0] && b[0] && a[0] == b[0]; len--, a++, b++) { + } + return (len == 0) ? 0 : (a[0] - b[0]); +# endif +#else + return strncmp(a, b, len); +#endif +} From 09be45df1246ad9ced494a1e0f7df8b130dfa650 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 16:39:55 +0000 Subject: [PATCH 02/46] Backport #125512 to 26.3: Never stop the server process from the line editor of the embedded client --- contrib/replxx | 2 +- src/Client/ReplxxLineReader.cpp | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/replxx b/contrib/replxx index 3a55a1bbee3c4..7ecd397fd2aea 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 3a55a1bbee3c4ccdbad38c0f1da4ce722e337807 +Subproject commit 7ecd397fd2aeabcad29c05fbf7ae8a2a83c3dede diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index df61d4542cf4e..14842dbfd77ef 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -342,7 +342,10 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) } } - rx.install_window_change_handler(); + /// The handler of `SIGWINCH` is process-wide. In the embedded mode the process is the server, + /// and the pseudoterminal is not its controlling terminal, so the signal is never delivered. + if (!options.embedded_mode) + rx.install_window_change_handler(); auto callback = [&suggest = options.suggest, this] (const String & context, size_t context_size) { @@ -365,7 +368,8 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); /// We don't want the default, "suspend" behavior, it confuses people. - if (options.ignore_shell_suspend) + /// In the embedded mode, it would stop the whole server process with `SIGSTOP`. + if (options.ignore_shell_suspend || options.embedded_mode) rx.bind_key_internal(replxx::Replxx::KEY::control('Z'), "insert_character"); auto commit_action = [this](char32_t code) From 183ca70deda1cf14df4eee309065ac83d3e36b1c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 16:45:30 +0000 Subject: [PATCH 03/46] Backport #125900 to 26.3: Fix stack overflow in the CapnProto schema compiler on long chains of declarations --- contrib/capnproto | 2 +- src/Formats/CapnProtoSchema.cpp | 7 ++ ...o_deep_schema_declaration_chains.reference | 17 +++++ ...apnproto_deep_schema_declaration_chains.sh | 75 +++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference create mode 100755 tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh diff --git a/contrib/capnproto b/contrib/capnproto index e7261205a6bea..7fababf324f50 160000 --- a/contrib/capnproto +++ b/contrib/capnproto @@ -1 +1 @@ -Subproject commit e7261205a6bea770d98e33ac94a35c2cd29e5b19 +Subproject commit 7fababf324f50e6a68be9415294ec2a4c3543208 diff --git a/src/Formats/CapnProtoSchema.cpp b/src/Formats/CapnProtoSchema.cpp index a793cf6a27c31..0353c4776842e 100644 --- a/src/Formats/CapnProtoSchema.cpp +++ b/src/Formats/CapnProtoSchema.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -182,6 +183,9 @@ void checkCapnProtoSchemaNestingDepth(const String & schema_directory, const Str capnp::StructSchema CapnProtoSchemaParser::getMessageSchema(const FormatSchemaInfo & schema_info) { + /// Bounds the compiler's recursion over chains of declarations, which the text check cannot see. + [[maybe_unused]] static const bool stack_check_installed = (capnp::compiler::setStackCheck(&checkStackSize), true); + checkCapnProtoSchemaNestingDepth(schema_info.schemaDirectory(), schema_info.schemaPath()); capnp::ParsedSchema schema; @@ -197,6 +201,9 @@ capnp::StructSchema CapnProtoSchemaParser::getMessageSchema(const FormatSchemaIn /// That's not good to determine the type of error by its description, but /// this is the only way to do it here, because kj doesn't specify the type of error. auto description = std::string_view(e.getDescription().cStr()); + if (e.getType() == kj::Exception::Type::OVERLOADED && description.contains("nested too deeply")) + throw Exception(ErrorCodes::CANNOT_PARSE_CAPN_PROTO_SCHEMA, "The CapnProto schema is nested too deeply"); + if (description.contains("No such file or directory") || description.contains("no such directory") || description.contains("no such file")) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot open CapnProto schema, file {} doesn't exists", schema_info.absoluteSchemaPath()); diff --git a/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference new file mode 100644 index 0000000000000..ef3f1a345d487 --- /dev/null +++ b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.reference @@ -0,0 +1,17 @@ +member_chain +1 +application_chain +1 +nested_suffix_chains +1 +alias_chain +1 +const_chain +1 +struct_chain_with_defaults +1 +enum_annotation_chain +1 +data Int32 +s Tuple(\n f Tuple(\n f Tuple(\n x Int32))) +e Enum8(\'v\' = 0) diff --git a/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh new file mode 100755 index 0000000000000..566c44d4fd8df --- /dev/null +++ b/tests/queries/0_stateless/05397_capnproto_deep_schema_declaration_chains.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Long chains of type expression suffixes or of declarations that refer to each other have no deep bracket nesting, +# but the Cap'n Proto schema compiler recurses once per link. They must be rejected, not overflow the stack. + +guard_message_count() { + local output count + output=$(cat) + count=$(grep -c -E 'Code: 622\..*nested too deeply' <<< "$output") || true + echo "$count" + [ "$count" = 1 ] || echo "FAIL: the depth guard did not fire, got: $(head -c 1000 <<< "$output" | tr '\n' ' ')" +} + +SCHEMA_DIR="$(mktemp -d "${CLICKHOUSE_TMP}/capnproto_schemas_XXXXXX")" +SCHEMA_DIR="$(cd "${SCHEMA_DIR}" && pwd)" +trap 'rm -rf "${SCHEMA_DIR}"' EXIT + +python3 - "${SCHEMA_DIR}" <<'PY' +import sys +directory = sys.argv[1] +header = '@0x844f048b15c12dab;\n' +message = 'struct M { data @0 :Int32; }\n' + +def alias_chain(n): + return ''.join(f'using A{i} = A{i + 1};\n' for i in range(n)) + f'using A{n} = Int32;\n' + +def const_chain(n): + return ''.join(f'const c{i} :Int32 = .c{i + 1};\n' for i in range(n)) + f'const c{n} :Int32 = 1;\n' + +def struct_chain(n): + return ''.join(f'struct S{i} {{ f @0 :S{i + 1}; }}\n' for i in range(n)) + f'struct S{n} {{ x @0 :Int32; }}\n' + +# P's default values load every S{i} first, so only walking the chain from P.d0 goes deep. +def struct_chain_with_defaults(n): + return 'struct P { ' + ' '.join(f'd{i} @{i} :S{i} = ();' for i in range(n + 1)) + ' }\n' + struct_chain(n) + +def enum_annotation_chain(n): + return ''.join(f'annotation t{i}(*) :E{i + 1};\nenum E{i} {{ v @0 $t{i}(v); }}\n' for i in range(n)) + f'enum E{n} {{ v @0; }}\n' + +def schema(name, text): + with open(directory + '/' + name + '.capnp', 'w') as f: + f.write(header + text) + +# Long enough to overflow the stack without the guard, and several times the length at which a release build rejects them. +n = 5000 +n_suffix = 20000 +schema('member_chain', 'struct M { data @0 :Int32' + '.a' * n_suffix + '; }\n') +schema('application_chain', 'struct M { data @0 :Int32' + '()' * n_suffix + '; }\n') +schema('nested_suffix_chains', 'struct M { data @0 :' + 'List(' * 10 + 'Int32' + (')' + '.a' * 200) * 10 + '; }\n') +schema('alias_chain', 'struct M { data @0 :A0; }\n' + alias_chain(n)) +schema('const_chain', message + const_chain(n)) +# A struct field costs less stack per link than the chains above. +schema('struct_chain_with_defaults', message + struct_chain_with_defaults(12000)) +schema('enum_annotation_chain', message + enum_annotation_chain(n)) +schema('shallow_chains', 'struct M { data @0 :A0; s @1 :S0; e @2 :E0; }\n' + + alias_chain(2) + const_chain(2) + struct_chain_with_defaults(2) + enum_annotation_chain(2)) +PY + +for name in member_chain application_chain nested_suffix_chains alias_chain const_chain struct_chain_with_defaults enum_annotation_chain +do + echo "${name}" + ${CLICKHOUSE_LOCAL} --logger.console=0 --query " + DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/${name}.capnp:M' + " 2>&1 | guard_message_count +done + +# Short chains of the same shapes are valid schemas. +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/shallow_chains.capnp:M' +" From e846d226eb897df36b4752f8994415ee55820cf6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 15 Oct 2026 17:57:29 +0000 Subject: [PATCH 04/46] Backport #119368 to 26.3: Do not deserialize unexpected packets in TCPHandler --- src/Server/TCPHandler.cpp | 164 +++--- src/Server/TCPHandler.h | 18 +- ...nexpected_packet_no_deserialization.python | 501 ++++++++++++++++++ ...pected_packet_no_deserialization.reference | 14 + ...75_unexpected_packet_no_deserialization.sh | 16 + 5 files changed, 601 insertions(+), 112 deletions(-) create mode 100644 tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python create mode 100644 tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.reference create mode 100755 tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.sh diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 2ee1d337a5814..4bb9399d5757c 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -252,6 +252,21 @@ void validateClientInfo(const ClientInfo & session_client_info, const ClientInfo // os_user, quota_key, client_trace_context can be different. } } + +/// Reject a packet that cannot appear at this point of the protocol without reading its payload. +/// `UNEXPECTED_PACKET_FROM_CLIENT` closes the connection, so the payload is never needed - and +/// deserializing it would expose a large amount of code (data types, serializations and aggregate +/// function states in a `Data` packet) to a peer that, in interserver mode, has not authenticated yet. +/// The unread payload is also why the connection cannot be reused: the leftover input starts +/// mid-packet, so `runImpl` does not try to skip it for this error code. +[[noreturn]] void throwUnexpectedPacket(UInt64 packet_type) +{ + throw Exception( + DB::ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, + "Unexpected packet {} received from client", + Protocol::Client::toString(packet_type)); +} + struct TurnOffBoolSettingTemporary { bool & setting; @@ -1032,8 +1047,11 @@ void TCPHandler::runImpl() std::lock_guard lock(*callback_mutex); /// A query packet is always followed by one or more data packets. - /// If some of those data packets are left, try to skip them. - if (!query_state->read_all_data) + /// If some of those data packets are left, try to skip them - but only from a packet + /// boundary. Skipping from inside a packet parses its remaining bytes as packets of + /// their own, deserializing whatever they say and then blocking until `receive_timeout`. + if (!query_state->read_all_data && query_state->at_packet_boundary + && exception_code != ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT) skipData(*query_state); } catch (...) @@ -1044,9 +1062,10 @@ void TCPHandler::runImpl() } /// We close the connection after an exception if there is something wrong with the connection, - /// otherwise we try to preserve it and reuse for other queries. - if (exception->code() == ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT || exception->code() == ErrorCodes::USER_EXPIRED - || exception->code() == ErrorCodes::TCP_CONNECTION_LIMIT_REACHED) + /// otherwise we try to preserve it and reuse for other queries. A half-read packet counts + /// as wrong: the next query would read the rest of it as a packet of its own. + if (!query_state->at_packet_boundary || exception_code == ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT + || exception_code == ErrorCodes::USER_EXPIRED || exception_code == ErrorCodes::TCP_CONNECTION_LIMIT_REACHED) { LOG_DEBUG(log, "Going to close connection due to exception: {}", exception->message()); query_state->finalizeOut(out); @@ -1104,7 +1123,7 @@ bool TCPHandler::receivePacketsExpectQuery(std::shared_ptr & state) switch (packet_type) { case Protocol::Client::Hello: - processUnexpectedHello(); + throwUnexpectedPacket(packet_type); case Protocol::Client::Data: case Protocol::Client::Scalar: @@ -1117,7 +1136,7 @@ bool TCPHandler::receivePacketsExpectQuery(std::shared_ptr & state) if (is_interserver_mode && !is_interserver_authenticated) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Unexpected data packet received before interserver authentication"); - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Data received from client"); + throwUnexpectedPacket(packet_type); case Protocol::Client::Ping: writeVarUInt(Protocol::Server::Pong, *out); @@ -1179,28 +1198,24 @@ bool TCPHandler::receivePacketsExpectData(QueryState & state) } UInt64 packet_type = 0; + /// The type is a varuint, so a client can leave the input mid-packet by truncating it. + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::IgnoredPartUUIDs: - processUnexpectedIgnoredPartUUIDs(); - case Protocol::Client::Query: - processUnexpectedQuery(); - case Protocol::Client::Hello: - processUnexpectedHello(); - case Protocol::Client::TablesStatusRequest: - processUnexpectedTablesStatusRequest(); + throwUnexpectedPacket(packet_type); case Protocol::Client::Data: case Protocol::Client::Scalar: { bool empty_block; if (state.skipping_data) - empty_block = !processUnexpectedData(); + empty_block = !skipDataPacket(state); else empty_block = !processData(state, packet_type == Protocol::Client::Scalar); if (empty_block) @@ -1212,12 +1227,14 @@ bool TCPHandler::receivePacketsExpectData(QueryState & state) return receiveQueryPlan(state); case Protocol::Client::Ping: + state.at_packet_boundary = true; /// no body writeVarUInt(Protocol::Server::Pong, *out); out->finishChunk(); out->next(); continue; case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return false; // We return false from this function as if no more data received @@ -1608,15 +1625,6 @@ void TCPHandler::processTablesStatusRequest() } -void TCPHandler::processUnexpectedTablesStatusRequest() -{ - TablesStatusRequest skip_request; - skip_request.read(*in, client_tcp_protocol_version); - - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet TablesStatusRequest received from client"); -} - - void TCPHandler::sendPartUUIDs(QueryState & state) { auto uuids = state.query_context->getPartUUIDs()->get(); @@ -2094,23 +2102,6 @@ void TCPHandler::receiveAddendum() } -void TCPHandler::processUnexpectedHello() -{ - UInt64 skip_uint_64; - String skip_string; - - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - readVarUInt(skip_uint_64, *in); - readVarUInt(skip_uint_64, *in); - readVarUInt(skip_uint_64, *in); - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Hello received from client"); -} - - void TCPHandler::sendHello() { writeVarUInt(Protocol::Server::Hello, *out); @@ -2179,22 +2170,16 @@ void TCPHandler::processIgnoredPartUUIDs() } -void TCPHandler::processUnexpectedIgnoredPartUUIDs() -{ - std::vector skip_part_uuids; - readVectorBinary(skip_part_uuids, *in); - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet IgnoredPartUUIDs received from client"); -} - - ClusterFunctionReadTaskResponsePtr TCPHandler::receiveClusterFunctionReadTaskResponse(QueryState & state) { UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return {}; @@ -2202,6 +2187,7 @@ ClusterFunctionReadTaskResponsePtr TCPHandler::receiveClusterFunctionReadTaskRes { auto task = std::make_shared(); task->deserialize(*in); + state.at_packet_boundary = true; return task; } @@ -2215,11 +2201,13 @@ ClusterFunctionReadTaskResponsePtr TCPHandler::receiveClusterFunctionReadTaskRes std::optional TCPHandler::receivePartitionMergeTreeReadTaskResponse(QueryState & state) { UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); return {}; @@ -2227,6 +2215,7 @@ std::optional TCPHandler::receivePartitionMergeTreeReadTas { ParallelReadResponse response; response.deserialize(*in, client_parallel_replicas_protocol_version); + state.at_packet_boundary = true; return response; } @@ -2307,7 +2296,6 @@ void TCPHandler::processQuery(std::shared_ptr & state) readVarUInt(compression, *in); state->compression = static_cast(compression); - last_block_in.compression = state->compression; readStringBinary(state->query, *in); @@ -2315,6 +2303,11 @@ void TCPHandler::processQuery(std::shared_ptr & state) if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) passed_params.read(*in, settings_format); + /// The packet is read, so the data packets that always follow it are owed from here on, whether + /// or not the rest of the query is set up successfully. Marking it any later would strand that + /// data for the next query on this connection to read as its first packet. + state->read_all_data = false; + if (is_interserver_mode) { client_info.interface = ClientInfo::Interface::TCP_INTERSERVER; @@ -2506,59 +2499,24 @@ void TCPHandler::processQuery(std::shared_ptr & state) std::chrono::milliseconds ms(sleep_after_receiving_query.totalMilliseconds()); std::this_thread::sleep_for(ms); } - - state->read_all_data = false; } -void TCPHandler::processUnexpectedQuery() -{ - UInt64 skip_uint_64; - String skip_string; - - readStringBinary(skip_string, *in, MAX_HELLO_STRING_SIZE); - - ClientInfo skip_client_info; - if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) - skip_client_info.read(*in, client_tcp_protocol_version); - - Settings skip_settings; - auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS - : SettingsWriteFormat::BINARY; - skip_settings.read(*in, settings_format); - - /// The same layout as in `processQuery`: the externally granted roles precede the interserver hash. - if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INTERSERVER_EXTERNALLY_GRANTED_ROLES) - readStringBinary(skip_string, *in, MAX_EXTRA_ROLES_SIZE); - - std::string skip_hash; - bool interserver_secret = client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET; - if (interserver_secret) - readStringBinary(skip_hash, *in, 32); - - readVarUInt(skip_uint_64, *in); - - readVarUInt(skip_uint_64, *in); - last_block_in.compression = static_cast(skip_uint_64); - - readStringBinary(skip_string, *in); - - if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) - skip_settings.read(*in, settings_format); - - throw Exception(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet Query received from client"); -} bool TCPHandler::receiveQueryPlan(QueryState & state) { - bool unexpected_packet = state.stage != QueryProcessingStage::QueryPlan || state.plan_and_sets || !state.query_context || state.read_all_data; - auto context = unexpected_packet ? Context::getGlobalContextInstance() : state.query_context; + /// Rejected while draining a failed query too: consuming it would only buy the reuse of a + /// connection whose query is already lost, at the price of running the plan deserializer on a + /// peer's bytes. + if (state.skipping_data || state.stage != QueryProcessingStage::QueryPlan || state.plan_and_sets + || !state.query_context || state.read_all_data) + throwUnexpectedPacket(Protocol::Client::QueryPlan); + + const auto & context = state.query_context; auto plan_and_sets = QueryPlan::deserialize(*in, context); + state.at_packet_boundary = true; LOG_TRACE(log, "Received query plan"); - if (!state.skipping_data && unexpected_packet) - throw NetException(ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT, "Unexpected packet QueryPlan received from client"); - state.plan_and_sets = std::make_shared(std::move(plan_and_sets)); return true; } @@ -2573,6 +2531,7 @@ bool TCPHandler::processData(QueryState & state, bool scalar) /// Read one block from the network and write it down Block block = state.block_in->read(); + state.at_packet_boundary = true; if (block.empty()) return false; @@ -2639,20 +2598,25 @@ bool TCPHandler::processData(QueryState & state, bool scalar) } -bool TCPHandler::processUnexpectedData() +bool TCPHandler::skipDataPacket(QueryState & state) { + /// The client keeps sending the data of a query that has already failed. The block is + /// deserialized only to find where the packet ends, so that the rest of the data can be + /// discarded and the connection reused for the next query. String skip_external_table_name; readStringBinary(skip_external_table_name, *in); std::shared_ptr maybe_compressed_in; - if (last_block_in.compression == Protocol::Compression::Enable) + if (state.compression == Protocol::Compression::Enable) maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true, /* external_data */ query_kind != ClientInfo::QueryKind::SECONDARY_QUERY); else maybe_compressed_in = in; - auto skip_block_in = std::make_shared(*maybe_compressed_in, client_tcp_protocol_version); - bool empty_block = skip_block_in->read().empty(); - return !empty_block; + NativeReader skip_block_in(*maybe_compressed_in, client_tcp_protocol_version); + bool has_rows = !skip_block_in.read().empty(); + + state.at_packet_boundary = true; + return has_rows; } @@ -2826,11 +2790,13 @@ void TCPHandler::receivePacketsExpectCancel(QueryState & state) throw NetException(ErrorCodes::ABORTED, "Client has dropped the connection, cancel the query."); UInt64 packet_type = 0; + state.at_packet_boundary = false; readVarUInt(packet_type, *in); switch (packet_type) { case Protocol::Client::Cancel: + state.at_packet_boundary = true; /// no body, and `processCancel` throws processCancel(state); break; diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 3819c87fb00a2..a3f89010be59c 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -121,6 +121,10 @@ struct QueryState /// If true, the data packets will be skipped instead of reading. Used to recover after errors. bool skipping_data = false; + /// Whether the input is positioned at the start of a packet. Cleared before the packet type is + /// read and set again once the packet is fully consumed, so any throw in between leaves it + /// cleared. Input starting mid-packet can only be discarded with the connection. + bool at_packet_boundary = true; bool query_duration_already_logged = false; ProfileEvents::ThreadIdToCountersSnapshot last_sent_snapshots; @@ -153,11 +157,6 @@ struct QueryState }; -struct LastBlockInputParameters -{ - Protocol::Compression compression = Protocol::Compression::Disable; -}; - class TCPHandler : public Poco::Net::TCPServerConnection { public: @@ -261,9 +260,6 @@ class TCPHandler : public Poco::Net::TCPServerConnection /// All the methods which are run inside callbacks are marked with TSA_REQUIRES. std::shared_ptr callback_mutex = std::make_shared(); - /// Last block input parameters are saved to be able to receive unexpected data packet sent after exception. - LastBlockInputParameters last_block_in; - CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection}; /// It is the name of the server that will be sent to the client. @@ -299,12 +295,8 @@ class TCPHandler : public Poco::Net::TCPServerConnection void readTemporaryTables(QueryState & state) TSA_REQUIRES(callback_mutex); void skipData(QueryState & state) TSA_REQUIRES(callback_mutex); + bool skipDataPacket(QueryState & state) TSA_REQUIRES(callback_mutex); - bool processUnexpectedData(); - [[noreturn]] void processUnexpectedQuery(); - [[noreturn]] void processUnexpectedIgnoredPartUUIDs(); - [[noreturn]] void processUnexpectedHello(); - [[noreturn]] void processUnexpectedTablesStatusRequest(); /// Process INSERT query void startInsertQuery(QueryState & state); diff --git a/tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python b/tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python new file mode 100644 index 0000000000000..c3c2edd7669e3 --- /dev/null +++ b/tests/queries/0_stateless/05175_unexpected_packet_no_deserialization.python @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# A packet that cannot appear at the current point of the native TCP protocol must be rejected with +# `UNEXPECTED_PACKET_FROM_CLIENT` (101) without reading its payload off the wire. +# +# Deserializing a payload only to throw it away exposes a large amount of code - data type parsing, +# serializations and aggregate function states in a `Data` packet - to a peer that, in interserver +# mode, has not authenticated yet. Every scenario below therefore sends the packet type and nothing +# else: a server that reads the payload waits for the missing bytes until the receive timeout, so +# the socket timeout here fails the test instead of the server answering right away. +# +# An unread payload cannot be told apart from the next packet, so the connection cannot be reused +# either: every scenario keeps the socket open and requires the server to close it. A server that +# drains the leftover input instead parses that payload as a packet stream of its own and blocks. +# +# The client speaks the raw native protocol pinned to revision 54482, so the set of exchanged, +# revision-gated fields is fixed and does not drift. + +import os +import socket +import struct +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "unexpected packet" + +REVISION = 54482 + +UNEXPECTED_PACKET_FROM_CLIENT = 101 +VIOLATED_CONSTRAINT = 469 + +UNKNOWN_TYPE_NAME = "ThisTypeDoesNotExist" + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt8(s): + return readUInt(s) + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) # database + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version (>= 54471) + readStringBinary(s) # timezone (>= 54058) + readStringBinary(s) # display name (>= 54372) + readVarUInt(s) # version patch (>= 54401) + readStringBinary(s) # proto_send chunked (>= 54470) + readStringBinary(s) # proto_recv chunked (>= 54470) + for _ in range(readVarUInt(s)): # password complexity rules (>= 54461) + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce, UInt64 (>= 54462) + # Server settings in STRINGS_WITH_FLAGS format: (name, flags, value)* terminated + # by an empty name (>= 54474). The server sends its changed settings here. + while True: + if readStringBinary(s) == "": # setting name + break + readVarUInt(s) # flags + readStringBinary(s) # value + readVarUInt(s) # query plan serialization version (>= 54477) + readVarUInt(s) # cluster function protocol version (>= 54479) + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key (>= 54458) + writeStringBinary("notchunked", ba) # proto_send chunked (>= 54470) + writeStringBinary("notchunked", ba) # proto_recv chunked (>= 54470) + writeVarUInt(0, ba) # parallel replicas protocol version (>= 54471) + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds (>= 54449) + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key (>= 54060) + writeVarUInt(0, ba) # distributed_depth (>= 54448) + writeVarUInt(1, ba) # client version patch (>= 54401) + ba.append(0) # opentelemetry: no trace id (>= 54442) + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator (>= 54453) + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number (>= 54475) + writeVarUInt(0, ba) # script line number (>= 54475) + ba.append(0) # jwt: none (>= 54476) + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings (they are set in the query text) + writeStringBinary("", ba) # interserver externally granted roles (>= 54472) + writeStringBinary("", ba) # interserver secret (>= 54441) + writeVarUInt(2, ba) # stage = Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator (>= 54459) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) # field 1 + ba.append(0) # is_overflows = false + writeVarUInt(2, ba) # field 2 + ba.extend(struct.pack("= 54480) + for _ in range(readVarUInt(s)): + readUInt32(s) + else: + raise RuntimeError("Unknown BlockInfo field {}".format(field_num)) + + +def readKindStack(s): + kind = readUInt8(s) + if kind == 5: # COMBINATION: number of kinds and the kinds themselves + readStrict(s, readVarUInt(s)) + + +def sendEmptyBlock(s): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) + serializeBlockInfo(ba) + writeVarUInt(0, ba) # columns + writeVarUInt(0, ba) # rows + s.sendall(ba) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readUInt8(s) # has_nested + return code, text + + +def readHeader(s): + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + code, text = readException(s) + raise RuntimeError("Unexpected exception {}: {}".format(code, text)) + assert packet_type == 1, "Expected Data (header) packet, got {}".format(packet_type) + readStringBinary(s) # external table name + readBlockInfo(s) + columns = readVarUInt(s) + rows = readVarUInt(s) + assert rows == 0, "Expected an empty header block, got {} rows".format(rows) + # Consume the per-column descriptor so the stream stays aligned. + for _ in range(columns): + readStringBinary(s) # column name + readStringBinary(s) # column type + if readUInt8(s): # has custom serialization (>= 54454) + readKindStack(s) + + +def readExceptionPacket(s, expected_code): + packet_type = readVarUInt(s) + assert packet_type == 2, "Expected Exception packet, got {}".format(packet_type) + code, text = readException(s) + assert code == expected_code, "Expected code {}, got {}: {}".format(expected_code, code, text) + return text + + +def connectAndHandshake(s): + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + receiveHello(s) + sendAddendum(s) + + +def sendPacketTypeOnly(s, packet_type, trailing=b""): + ba = bytearray() + writeVarUInt(packet_type, ba) + ba.extend(trailing) + s.sendall(ba) + + +def assertConnectionClosed(s): + # `socket.timeout` is not caught on purpose: it is how a server that keeps waiting for the + # payload of the rejected packet fails this test. + assert s.recv(1) == b"", "Expected the server to close the connection" + + +def sendUInt64Block(s, values): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) # external table name + serializeBlockInfo(ba) + writeVarUInt(1, ba) # columns + writeVarUInt(len(values), ba) # rows + writeStringBinary("x", ba) + writeStringBinary("UInt64", ba) + ba.append(0) # no custom serialization + for value in values: + ba.extend(struct.pack(" Date: Fri, 16 Oct 2026 08:13:08 +0000 Subject: [PATCH 05/46] Backport #125221 to 26.3: Fix `ORDER BY ... LIMIT` over an `Interval` column with top-K dynamic filtering --- src/Functions/FunctionTopKFilter.cpp | 8 +- .../QueryPlan/Optimizations/optimizeTopK.cpp | 2 +- src/Processors/TopKThresholdTracker.h | 8 +- ...61_top_k_dynamic_filter_interval.reference | 120 ++++++++++++++++++ .../05361_top_k_dynamic_filter_interval.sql | 107 ++++++++++++++++ 5 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference create mode 100644 tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql diff --git a/src/Functions/FunctionTopKFilter.cpp b/src/Functions/FunctionTopKFilter.cpp index f2c2c434bd192..5755622c3e5e4 100644 --- a/src/Functions/FunctionTopKFilter.cpp +++ b/src/Functions/FunctionTopKFilter.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ class FunctionTopKFilter: public IFunction if (!threshold_tracker_) throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionTopKFilter got NULL threshold_tracker"); + threshold_type = recursiveRemoveLowCardinality(threshold_tracker_->getDataType()); String comparator = "lessOrEquals"; if (threshold_tracker->getDirection() == -1) /// DESC @@ -78,7 +80,9 @@ class FunctionTopKFilter: public IFunction auto current_threshold = threshold_tracker->getValue(); auto data_type = arguments[0].type; - ColumnPtr threshold_column = data_type->createColumnConst(input_rows_count, convertFieldToType(current_threshold, *data_type)); + /// The threshold is a value of the tracker's type, which a `Field` does not carry (an `Interval` is a plain `Int64`). + ColumnPtr threshold_column = data_type->createColumnConst( + input_rows_count, convertFieldToType(current_threshold, *data_type, threshold_type.get())); ColumnsWithTypeAndName args{ arguments[0], {threshold_column, data_type, {}} }; auto elem_compare = compare_function->build(args); @@ -93,6 +97,8 @@ class FunctionTopKFilter: public IFunction } private: TopKThresholdTrackerPtr threshold_tracker; + /// The type of the threshold value as this function receives it (the default implementation removes LowCardinality). + DataTypePtr threshold_type; FunctionOverloadResolverPtr compare_function; int direction; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp index 2e8fcb3b4b9f7..61180c08f7a52 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp @@ -129,7 +129,7 @@ size_t tryOptimizeTopK(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, read_from_mergetree_step->isSkipIndexAvailableForTopK(sort_column_name) && settings.use_skip_indexes_on_data_read) || (settings.use_top_k_dynamic_filtering && !read_from_mergetree_step->getPrewhereInfo())) { - threshold_tracker = std::make_shared(direction); + threshold_tracker = std::make_shared(direction, sort_column.type); sorting_step->setTopKThresholdTracker(threshold_tracker); } diff --git a/src/Processors/TopKThresholdTracker.h b/src/Processors/TopKThresholdTracker.h index ae38a4103921a..47b045539c6cd 100644 --- a/src/Processors/TopKThresholdTracker.h +++ b/src/Processors/TopKThresholdTracker.h @@ -12,9 +12,12 @@ namespace DB { +class IDataType; +using DataTypePtr = std::shared_ptr; + struct TopKThresholdTracker { - explicit TopKThresholdTracker(int direction_) : direction(direction_) {} + TopKThresholdTracker(int direction_, DataTypePtr data_type_) : direction(direction_), data_type(std::move(data_type_)) {} void testAndSet(const Field & value) { @@ -65,12 +68,15 @@ struct TopKThresholdTracker bool isSet() const { return is_set; } /// unlocked read is fine int getDirection() const { return direction; } + /// The type of the sort column the published values are taken from. + const DataTypePtr & getDataType() const { return data_type; } private: Field threshold; mutable SharedMutex mutex; std::atomic is_set{false}; int direction{0}; + DataTypePtr data_type; }; using TopKThresholdTrackerPtr = std::shared_ptr; diff --git a/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference new file mode 100644 index 0000000000000..d0c6d8ca502b4 --- /dev/null +++ b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.reference @@ -0,0 +1,120 @@ +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +\N +\N +0 +0 +1 +2 +\N +\N +99999 +99999 +99998 +99997 +\N +\N +0 +0 +1 +2 +\N +\N +99999 +99999 +99998 +99997 +(0,0) +(0,1) +(0,2) +(1666,39) +(1666,38) +(1666,37) +99999 +99998 +99997 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +0 +1 +2 +99999 +99998 +99997 +\N +\N +0 +99999 +99998 +99997 diff --git a/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql new file mode 100644 index 0000000000000..92d2abf7c3642 --- /dev/null +++ b/tests/queries/0_stateless/05361_top_k_dynamic_filter_interval.sql @@ -0,0 +1,107 @@ +-- Random settings limits: use_top_k_dynamic_filtering=(1, 1); query_plan_max_limit_for_top_k_optimization=(1000, None); max_block_size=(None, 8192) + +-- `ORDER BY ... LIMIT` over a column whose type is or contains an `Interval` type, with the top-K dynamic filter. + +SET use_top_k_dynamic_filtering = 1; + +DROP TABLE IF EXISTS t_interval; +DROP TABLE IF EXISTS t_interval_key; +DROP TABLE IF EXISTS t_interval_minmax; +DROP TABLE IF EXISTS t_interval_memory; +DROP TABLE IF EXISTS t_interval_lc; + +CREATE TABLE t_interval +( + id UInt64, + d_ns IntervalNanosecond, + d_us IntervalMicrosecond, + d_ms IntervalMillisecond, + d_s IntervalSecond, + d_min IntervalMinute, + d_h IntervalHour, + d_d IntervalDay, + d_w IntervalWeek, + d_mon IntervalMonth, + d_q IntervalQuarter, + d_y IntervalYear, + n_s Nullable(IntervalSecond), + n_ns Nullable(IntervalNanosecond), + tup Tuple(IntervalHour, IntervalMinute), + arr Array(IntervalSecond) +) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO t_interval SELECT + number, + toIntervalNanosecond(number), toIntervalMicrosecond(number), toIntervalMillisecond(number), toIntervalSecond(number), + toIntervalMinute(number), toIntervalHour(number), toIntervalDay(number), toIntervalWeek(number), + toIntervalMonth(number), toIntervalQuarter(number), toIntervalYear(number), + if(number % 50000 = 7, NULL, toIntervalSecond(number)), + if(number % 50000 = 7, NULL, toIntervalNanosecond(number)), + (toIntervalHour(intDiv(number, 60)), toIntervalMinute(number % 60)), + [toIntervalSecond(number)] +FROM numbers(100000); + +SELECT d_ns FROM t_interval ORDER BY d_ns LIMIT 3; +SELECT d_ns FROM t_interval ORDER BY d_ns DESC LIMIT 3; +SELECT d_us FROM t_interval ORDER BY d_us LIMIT 3; +SELECT d_us FROM t_interval ORDER BY d_us DESC LIMIT 3; +SELECT d_ms FROM t_interval ORDER BY d_ms LIMIT 3; +SELECT d_ms FROM t_interval ORDER BY d_ms DESC LIMIT 3; +SELECT d_s FROM t_interval ORDER BY d_s LIMIT 3; +SELECT d_s FROM t_interval ORDER BY d_s DESC LIMIT 3; +SELECT d_min FROM t_interval ORDER BY d_min LIMIT 3; +SELECT d_min FROM t_interval ORDER BY d_min DESC LIMIT 3; +SELECT d_h FROM t_interval ORDER BY d_h LIMIT 3; +SELECT d_h FROM t_interval ORDER BY d_h DESC LIMIT 3; +SELECT d_d FROM t_interval ORDER BY d_d LIMIT 3; +SELECT d_d FROM t_interval ORDER BY d_d DESC LIMIT 3; +SELECT d_w FROM t_interval ORDER BY d_w LIMIT 3; +SELECT d_w FROM t_interval ORDER BY d_w DESC LIMIT 3; +SELECT d_mon FROM t_interval ORDER BY d_mon LIMIT 3; +SELECT d_mon FROM t_interval ORDER BY d_mon DESC LIMIT 3; +SELECT d_q FROM t_interval ORDER BY d_q LIMIT 3; +SELECT d_q FROM t_interval ORDER BY d_q DESC LIMIT 3; +SELECT d_y FROM t_interval ORDER BY d_y LIMIT 3; +SELECT d_y FROM t_interval ORDER BY d_y DESC LIMIT 3; + +SELECT n_s FROM t_interval ORDER BY n_s ASC NULLS FIRST LIMIT 3; +SELECT n_s FROM t_interval ORDER BY n_s ASC NULLS LAST LIMIT 3; +SELECT n_s FROM t_interval ORDER BY n_s DESC NULLS FIRST LIMIT 3; +SELECT n_s FROM t_interval ORDER BY n_s DESC NULLS LAST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns ASC NULLS FIRST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns ASC NULLS LAST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns DESC NULLS FIRST LIMIT 3; +SELECT n_ns FROM t_interval ORDER BY n_ns DESC NULLS LAST LIMIT 3; + +SELECT tup FROM t_interval ORDER BY tup LIMIT 3; +SELECT tup FROM t_interval ORDER BY tup DESC LIMIT 3; + +CREATE TABLE t_interval_key (d IntervalSecond) ENGINE = MergeTree ORDER BY d; +INSERT INTO t_interval_key SELECT toIntervalSecond(number) FROM numbers(100000); +SELECT d FROM t_interval_key ORDER BY d DESC LIMIT 3 SETTINGS optimize_read_in_order = 0; + +CREATE TABLE t_interval_minmax (id UInt64, d IntervalSecond, INDEX idx d TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_interval_minmax SELECT number, toIntervalSecond(number) FROM numbers(100000); +SELECT d FROM t_interval_minmax ORDER BY d DESC LIMIT 3 SETTINGS use_skip_indexes_for_top_k = 1, use_skip_indexes_on_data_read = 1; + +CREATE TABLE t_interval_memory (d IntervalSecond) ENGINE = Memory; +INSERT INTO t_interval_memory SELECT toIntervalSecond(number) FROM numbers(100000); +SELECT d FROM t_interval_memory ORDER BY d LIMIT 3; +SELECT d FROM t_interval_memory ORDER BY d DESC LIMIT 3; + +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE t_interval_lc (id UInt64, d LowCardinality(IntervalSecond), n LowCardinality(Nullable(IntervalSecond))) +ENGINE = MergeTree ORDER BY id; +INSERT INTO t_interval_lc SELECT number, toIntervalSecond(number), if(number % 50000 = 7, NULL, toIntervalSecond(number)) +FROM numbers(100000); +SELECT d FROM t_interval_lc ORDER BY d LIMIT 3; +SELECT d FROM t_interval_lc ORDER BY d DESC LIMIT 3; +SELECT n FROM t_interval_lc ORDER BY n ASC NULLS FIRST LIMIT 3; +SELECT n FROM t_interval_lc ORDER BY n DESC NULLS LAST LIMIT 3; + +DROP TABLE t_interval; +DROP TABLE t_interval_key; +DROP TABLE t_interval_minmax; +DROP TABLE t_interval_memory; +DROP TABLE t_interval_lc; From 7792fa0d0e2ec7d4dd3d15ca4740c8d0fd381c87 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 11:54:21 +0000 Subject: [PATCH 06/46] Backport #121704 to 26.3: Stop the native SSH-key handshake from disclosing whether a user exists --- src/Server/TCPHandler.cpp | 13 -- .../0_stateless/03780_failed_ssh.expect | 61 ++++-- .../04908_ssh_key_auth_user_enumeration.py | 186 ++++++++++++++++++ ...08_ssh_key_auth_user_enumeration.reference | 2 + 4 files changed, 235 insertions(+), 27 deletions(-) create mode 100755 tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py create mode 100644 tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 4bb9399d5757c..1d7b6d09bf162 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2025,19 +2025,6 @@ void TCPHandler::receiveHello() /// Perform handshake for SSH authentication if (is_ssh_based_auth) { - const auto authentication_types = session->getAuthenticationTypesOrLogInFailure(user); - - bool user_supports_ssh_authentication = std::find_if( - authentication_types.begin(), - authentication_types.end(), - [](auto authentication_type) - { - return authentication_type == AuthenticationType::SSH_KEY; - }) != authentication_types.end(); - - if (!user_supports_ssh_authentication) - throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Expected authentication with SSH key"); - if (client_tcp_protocol_version < DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Cannot authenticate user with SSH key, because client version is too old"); diff --git a/tests/queries/0_stateless/03780_failed_ssh.expect b/tests/queries/0_stateless/03780_failed_ssh.expect index e1b638438fa22..c4bb68e7851ae 100755 --- a/tests/queries/0_stateless/03780_failed_ssh.expect +++ b/tests/queries/0_stateless/03780_failed_ssh.expect @@ -23,20 +23,53 @@ expect_after { } spawn bash -c "echo '-----BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAlwAAAAdzc2gtcn -NhAAAAAwEAAQAAAIEAxOYq7137PbwKYw6IahB/pHvYNvFTrMg6bAT182PLg+Fidb+wbMNw -qkeL4C8wIxwulaScbjpidEEVbB3egOXf92cVmO7FxnwJkbuOkQdTZ68+vh9oy7jRwLsTq0 -K5Ez0nnauHyoR2Qdd+7AnTSSyEi1Efdni5Ble/LhHhRPU//ckAAAIQRkn3H0ZJ9x8AAAAH -c3NoLXJzYQAAAIEAxOYq7137PbwKYw6IahB/pHvYNvFTrMg6bAT182PLg+Fidb+wbMNwqk -eL4C8wIxwulaScbjpidEEVbB3egOXf92cVmO7FxnwJkbuOkQdTZ68+vh9oy7jRwLsTq0K5 -Ez0nnauHyoR2Qdd+7AnTSSyEi1Efdni5Ble/LhHhRPU//ckAAAADAQABAAAAgDW3MSliM0 -NxREgn09FTtO7TlnXOumwdp8qGQ+7lX8UXvLuw0tmpK9hYcnFzjidV7eOHJ+HubaOXideY -AnaZv9KNNWBDTOeuHpMtaIzjq3yb/BuOSC1Xthjb7UZoQGlgAUOdCTFap3jxV8PeLbFazg -oMqjTVCwiWxlIumBoZgwTNAAAAQQCxyh6w3GOFtqts66TVNs8pSe8r/cSOUnTU1cws5s0t -l+CHod8qOHF+eWWnw+TvPw3sTM/5rcJ+0N3anhpo9Ph3AAAAQQDl0EKJvfyRgnKHcjtLUM -IPac9E4ZR36UA2lWK/gr0x8PvxzlonzWqBcYFmCAQv9lPvk1GJK8/EnCkkD5pTvKbHAAAA -QQDbVchcKjXrhsYklDvJQ4xyGkUH6+xMRVINwzjP3JQ1akUF/MJAYg5vr8j4gibWysVwWV -0lgWuG96Yy/MxT62bvAAAAE2lzYWtlbEBwYzY0MTAxLTI1MzYBAgMEBQYH +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEAoUTN2fewg1IpJFd006wAQ7O2Y9ntq5KUQ2Tpvmxd/hrrPY3PdQmt +Vvnws1Sn3pn1NWSKxYD+Yfh5ZRhioWv9lJe2IxgbvA1tEIgX0Sx5T5fbkCZMaOzMXFsEvp +KWPlBTkz557lvQQVOvzAHnH20LC4o84/QileP4rhQR86CtAoJkweLPv/+0nMTeEC9FPGwm +ls2u2ezz2mLfHKU1+vlrXa+qzfvyl8aWDdeR5VDfRQ6foXSoji6z1zlf1JSulrUq76DdMJ +1iH+Y8sEYbNpvVaW6nSqpIhrgg6Z1qY+sBIheTSyiX1IS9DeDrXMxi+NKGEmiGM+Rn+RKR +0V7fk8fJLlUSf5g3KDTjd7DxvS7Zu73kNR/ODUWpUWNw+/OlhGCseNQZgOhXdkF9m87wH0 +9wB+Ny5gH06z8A5jdi1k7h3+pD1Q/CV2tL80xrEga0wIeaytAigiqqc6FiyIKtDmUIr+2v +NR/iHW59tqPehG5kFV5+zuqVe8xHehV40tQza4LQaXDvR6US1u5Bk3zRE8p0ZHGF1VnRXR +vSgvMEJFYzEPIj99sCWUw3CD2NdmKRGI3kpDrCyMHbCwWR1HYhtazpMiauAxFxVsNHInCA +2HTar629DKpgzW3C3yQBPhRofg2kZlbQCS1MxZzaOlRpBKDnIkqicwlpUEARnL6QFnNlhD +8AAAc4P+ey+j/nsvoAAAAHc3NoLXJzYQAAAgEAoUTN2fewg1IpJFd006wAQ7O2Y9ntq5KU +Q2Tpvmxd/hrrPY3PdQmtVvnws1Sn3pn1NWSKxYD+Yfh5ZRhioWv9lJe2IxgbvA1tEIgX0S +x5T5fbkCZMaOzMXFsEvpKWPlBTkz557lvQQVOvzAHnH20LC4o84/QileP4rhQR86CtAoJk +weLPv/+0nMTeEC9FPGwmls2u2ezz2mLfHKU1+vlrXa+qzfvyl8aWDdeR5VDfRQ6foXSoji +6z1zlf1JSulrUq76DdMJ1iH+Y8sEYbNpvVaW6nSqpIhrgg6Z1qY+sBIheTSyiX1IS9DeDr +XMxi+NKGEmiGM+Rn+RKR0V7fk8fJLlUSf5g3KDTjd7DxvS7Zu73kNR/ODUWpUWNw+/OlhG +CseNQZgOhXdkF9m87wH09wB+Ny5gH06z8A5jdi1k7h3+pD1Q/CV2tL80xrEga0wIeaytAi +giqqc6FiyIKtDmUIr+2vNR/iHW59tqPehG5kFV5+zuqVe8xHehV40tQza4LQaXDvR6US1u +5Bk3zRE8p0ZHGF1VnRXRvSgvMEJFYzEPIj99sCWUw3CD2NdmKRGI3kpDrCyMHbCwWR1HYh +tazpMiauAxFxVsNHInCA2HTar629DKpgzW3C3yQBPhRofg2kZlbQCS1MxZzaOlRpBKDnIk +qicwlpUEARnL6QFnNlhD8AAAADAQABAAACAEaNMj1cReV9iTAz7pE32znDIlTWn0qDKse9 +GyFKiWhEYIuiQZMjM6ksWSgrLfGNix/bp3Y2M5Oqv3kBzQxvkRVztaPuxRtlCETGNR02hq +/ScKTEjIMOrWzuCMpSmNYUECwHlhI4KoiZ0vu+DV3LN5zJ0k+rm4HVpRKLBzt9HJNPdkL4 +Ngj6FgrLeZa9B14itlMdEpSWPCxbOt5uTIpKmbT3yBKrWXsazsSpfVNEKRkuB5n7zk29ks +HigNEgZqzZ/zmsN3gwMiBgekZntToTvK6OUTOyAV6mqQdq7YNe6TT/+VlVHe9ETu0GbBFB +/T5741mkTqnfD7mAbTiad9dP/OvND1c40v1n4e3ZHU/FtUwCArr2oKP0Q6+Y1ool5Szi/M +MUsjepnKqFuaLpkuDThaPLBissz+nP3HRlwqeaxFKTk/Hl4IraGK1BDVc1x4mQsFaobL/3 ++hxZf/fN+lDTHjr93OXqgU/yUK9jmp7n3KICZNZz6wyqD/bVL1WIGPxHdCnq77S7TYTH48 +XGErc0P9xsnXVlqE4Vxuw2bP6FPrQ8EzjGj53H5k0kBa3fospC6rwEUAB4g0W1aW5ZzU0D +EASqTnFEX33ZB06iBDLZqDEEl7VSKVDR3GBHBLJXAvQojmFHpRXAm3SXfugYTLfF+qmgnu +iNfYRQz+lMzxD8331hAAABABosTEeM32bFSQOIK5uzJUphbhhJvwpLv+6jzr+7GLYMy26q +QfFfmRK7qHHfTqHSngQ5h0H5AOrkLR/xw6vetDzKEuRnnDgsAPaT0wt8wKV66vSrmO6GFX +cYi/qg2D8YJjeMSyKGD8Rd6Nq+ukssQ9LpNCkbYL6Zom/Qvd2ZlQGkoh6iVdOX78ULbdN1 +XmJwjrhggIKJWSFrv4cjZEP4Q3g11FC+zs+R71/i0YDcnDiNbh+1khNfMDxpzaaV1L3bwE +ouyYlk17eQghM7rgpnh76/1dlI+/O+S9P6o7fQT1aIiG5ms4LOMDtFyV88nq2Q+3RXYhTT +ZGvlaC4L5weIcecAAAEBANbOknEph4Hc9kuWGOcH6TmbbJrCktE/jv66YlwbegS9q8krJ7 +6thAS6gNi3vojVpi2i4w8eNkttjEwNsg7WMVjzV9XTF8R6HbPn33s+EHaLiL1zn7wcmAIB +HNalc5zjC5pfOf6vfe4JyZNRUyoHUQUNzqJRuea8h7lmup07STCUWQOypmEZK6zsi7We6Y +pkl+byMBXFUifE5pW749uygtT3vR7djQiYGPevYIVKqw3LQ5R7QeZ4ZZJaTXbrD2dHWyr9 +DroHgfGMsfVHvLTUc+TmL+SrBNUOHftMxo4Szy2b2IPP5HWedDwD1ft5x2O98xU6jIJmnm +F5tk/sAv8CgG8AAAEBAMAx58ag2qGMcOv36DhhhrwMKlLQgb3Wqooo6FvR6TaxjqAu/8b4 +UFgi/KV4Irdrd22hvYJDNseJNKqCKGOwvPiHE3uu1ERmmrG/xw7NAlXZPDoHclK/tr/xPE +IEGOqRQHbHaCE/0N91ncF35Hu3dXQau0p+DszUt9RZpO4YllJEMe+Xy/sxlv3ul6H522ZL +a4ifUw559aqIXXvsn8iFT1iLt1t3nDjepptWHggOb4VbuEAY24HSbxW9OwTcEVEvlpXNg0 +kcintN4pJb/F3J1SGPIy/Hj6fLTHkMGcYqJz2oKMRTv/wOdBFCQay5juDeOJTXhyCtDPoV +V0u3f4V9gTEAAAAAAQID -----END OPENSSH PRIVATE KEY-----' > id_rsa_03780_\$CLICKHOUSE_DATABASE" spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CURL -sS \$CLICKHOUSE_URL -d \"CREATE USER IF NOT EXISTS user_03780_\$CLICKHOUSE_DATABASE\"" diff --git a/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py new file mode 100755 index 0000000000000..67dd8d3dd5b06 --- /dev/null +++ b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# Tags: no-fasttest, no-openssl-fips +# no-fasttest: the SSH handshake is only compiled into a server built with SSH support. +# no-openssl-fips: creates an ssh-ed25519 credential, which FIPS builds reject. +"""Test that SSH-key authentication does not reveal whether a user exists. + +A client asks for it by putting a marker in front of the user name, and the server answers with a +challenge to sign. Every name must get that far - one with an SSH key, one with a password, one +that nobody has - and all three must then fail identically apart from the name itself. + +The client asks for SSH-key authentication only when it holds a private key, so it cannot reach +the last two cases. This test speaks the native protocol over a raw socket instead, asking for a +challenge for each name and answering with garbage. +""" + +import os +import socket +import subprocess + +# `clickhouse-test` exports CLICKHOUSE_BINARY and CLICKHOUSE_DATABASE, but not the host and port - +# those come from the environment the suite was started in, and `shell_config.sh`, which supplies +# them, is only sourced by the `.sh` tests. Hence the defaults, as in the other `.py` tests here. +HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", 9000)) +DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT = [ + os.environ.get("CLICKHOUSE_BINARY", "clickhouse"), + "client", + "--host", + HOST, + "--port", + str(PORT), + "--database", + DATABASE, +] + +TIMEOUT = 20 + +# Packet types of the native protocol. +CLIENT_HELLO = 0 +CLIENT_SSH_CHALLENGE_REQUEST = 11 +CLIENT_SSH_CHALLENGE_RESPONSE = 12 +SERVER_EXCEPTION = 2 +SERVER_SSH_CHALLENGE = 18 + +# A user name behind this marker is what asks for SSH-key authentication. +SSH_KEY_MARKER = " SSH KEY AUTHENTICATION " + +# The oldest protocol revision that supports the SSH handshake; the server refuses older ones. +REVISION = 54466 + +AUTHENTICATION_FAILED = 516 +GENERIC_ERROR = ( + "Authentication failed: password is incorrect, or there is no user with such name" +) + +# The three user states that must be indistinguishable. The test database is unique per run, so +# naming the users after it keeps parallel runs apart. The password user is not `default`, whose +# error message carries extra password-reset help. +SSH_USER = f"ssh_user_{DATABASE}" +PASSWORD_USER = f"password_user_{DATABASE}" +UNKNOWN_USER = f"nosuchuser_{DATABASE}" + +# Any valid public key will do, because no signature the test sends can match it. +PUBLIC_KEY_FILE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "../../config/ssh_user_ed25519_key.pub" +) + + +def run_query(query): + subprocess.run(CLIENT + ["--query", query], check=True) + + +def write_varuint(value): + encoded = bytearray() + while value >= 0x80: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def write_string(value): + data = value.encode() + return write_varuint(len(data)) + data + + +def read_exact(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError(f"connection closed after {len(data)} of {size} bytes") + data += chunk + return data + + +def read_varuint(sock): + value = 0 + for i in range(9): + byte = read_exact(sock, 1)[0] + value |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + break + return value + + +def read_string(sock): + return read_exact(sock, read_varuint(sock)).decode() + + +def read_exception(sock): + # Code, name and message. The stack trace and the nested flag that follow are not read, + # because the connection is closed right after. + code = int.from_bytes(read_exact(sock, 4), "little", signed=True) + read_string(sock) + message = read_string(sock) + return code, message + + +def ssh_handshake(user): + """Request SSH-key authentication as `user` and answer the challenge with a bad signature. + + Returns whether the server asked for a signature at all, and the error it ended with. + """ + hello = ( + write_varuint(CLIENT_HELLO) + + write_string("probe") # client name + + write_varuint(24) # client version major + + write_varuint(3) # client version minor + + write_varuint(REVISION) + + write_string("") # default database + + write_string(SSH_KEY_MARKER + user) + + write_string("") # empty password + ) + + with socket.create_connection((HOST, PORT), timeout=TIMEOUT) as sock: + sock.sendall(hello + write_varuint(CLIENT_SSH_CHALLENGE_REQUEST)) # and ask for a challenge + + packet = read_varuint(sock) + if packet == SERVER_EXCEPTION: + code, message = read_exception(sock) + return False, code, message + assert packet == SERVER_SSH_CHALLENGE, f"unexpected packet {packet}" + + read_exact(sock, read_varuint(sock)) # the challenge, which the test cannot sign + sock.sendall( + write_varuint(CLIENT_SSH_CHALLENGE_RESPONSE) + write_string("not-a-signature") + ) + + packet = read_varuint(sock) + assert packet == SERVER_EXCEPTION, f"unexpected packet {packet}" + code, message = read_exception(sock) + return True, code, message + + +def main(): + with open(PUBLIC_KEY_FILE, encoding="utf-8") as key_file: + public_key = key_file.read().split()[1] + + run_query(f"DROP USER IF EXISTS {SSH_USER}, {PASSWORD_USER}") + try: + run_query( + f"CREATE USER {SSH_USER} IDENTIFIED WITH ssh_key BY KEY '{public_key}' TYPE 'ssh-ed25519'" + ) + run_query(f"CREATE USER {PASSWORD_USER} IDENTIFIED WITH sha256_password BY 'password'") + + errors = set() + for user in (SSH_USER, PASSWORD_USER, UNKNOWN_USER): + challenged, code, message = ssh_handshake(user) + # Being refused before the challenge is itself a disclosure, whatever the error says. + assert challenged, f"{user} was refused before the challenge: {message}" + assert code == AUTHENTICATION_FAILED, f"{user}: error {code}: {message}" + assert GENERIC_ERROR in message, f"{user}: unexpected error: {message}" + # The name is the one thing the caller already knows, so it may differ - nothing else. + errors.add(message.replace(user, "USER")) + finally: + run_query(f"DROP USER IF EXISTS {SSH_USER}, {PASSWORD_USER}") + + print("every user was challenged and then got the generic authentication error") + assert len(errors) == 1, f"the errors differ: {errors}" + print("the errors are identical apart from the user name") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference new file mode 100644 index 0000000000000..9106017f70ba1 --- /dev/null +++ b/tests/queries/0_stateless/04908_ssh_key_auth_user_enumeration.reference @@ -0,0 +1,2 @@ +every user was challenged and then got the generic authentication error +the errors are identical apart from the user name From 09d38b1fe393421b53a687752cad9acf61669770 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 16 Oct 2026 13:51:34 +0000 Subject: [PATCH 07/46] Update autogenerated version to 26.3.53.4 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 158c98901c11a..4f96015e94281 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54560) +SET(VERSION_REVISION 54561) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 53) -SET(VERSION_GITHASH fe8ff5d492ab7325e974b19a619b5c3ab3c095d8) -SET(VERSION_DESCRIBE v26.3.53.1-lts) -SET(VERSION_STRING 26.3.53.1) +SET(VERSION_PATCH 54) +SET(VERSION_GITHASH d8af9de0b15bce32bc1e9e4b48d3e234f2cff5cd) +SET(VERSION_DESCRIBE v26.3.54.1-lts) +SET(VERSION_STRING 26.3.54.1) # end of autochange From f7a8f8d9a57da0080f7bb65fd9875770915bcb4d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 03:21:27 +0000 Subject: [PATCH 08/46] Backport #125899 to 26.3: Apply `http_zlib_compression_level` only to zlib-based HTTP compression --- src/Core/FormatFactorySettings.h | 2 +- src/Core/Settings.cpp | 2 + src/Server/HTTPHandler.cpp | 12 +++++- ...zlib_compression_level_only_zlib.reference | 6 +++ ...8_http_zlib_compression_level_only_zlib.sh | 38 +++++++++++++++++++ 5 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference create mode 100755 tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index 0d2ea68f9fba3..0042be8fe8a10 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -663,7 +663,7 @@ Allow data types conversion in Native input format Read data types in binary format instead of type names in Native input format )", 0) \ DECLARE(UInt64, output_format_compression_level, 3, R"( -Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when writing to table functions `file`, `url`, `hdfs`, `s3`, or `azureBlobStorage`. +Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when writing to table functions `file`, `url`, `hdfs`, `s3`, or `azureBlobStorage`. It is also applied to the compressed response to an HTTP request with a method other than `gzip` and `deflate`, for which [http_zlib_compression_level](#http_zlib_compression_level) is used. Possible values: from `1` to `22` )", 0) \ diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 33a9e28705d45..ac119469ccd0e 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -2203,6 +2203,8 @@ Possible values: DECLARE(Int64, http_zlib_compression_level, 3, R"( Sets the level of data compression in the response to an HTTP request if [enable_http_compression = 1](#enable_http_compression). +The setting applies only to the zlib-based methods: `gzip` and `deflate`. The level for other methods (such as `zstd`, `lz4`, `br`, `xz`) is set by [output_format_compression_level](#output_format_compression_level). + Possible values: Numbers from 1 to 9. )", 0) \ \ diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 4792688794919..81da4aa5f30b2 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -68,6 +68,7 @@ namespace Setting extern const SettingsBool http_wait_end_of_query; extern const SettingsBool http_write_exception_in_output_format; extern const SettingsInt64 http_zlib_compression_level; + extern const SettingsUInt64 output_format_compression_level; extern const SettingsUInt64 readonly; extern const SettingsBool send_progress_in_http_headers; extern const SettingsInt64 zstd_window_log_max; @@ -349,6 +350,15 @@ void HTTPHandler::processQuery( bool enable_http_compression = params.getParsedLast("enable_http_compression", settings[Setting::enable_http_compression]); Int64 http_zlib_compression_level = params.getParsed("http_zlib_compression_level", settings[Setting::http_zlib_compression_level]); + /// `http_zlib_compression_level` is for the zlib-based methods only. Other methods have their own ranges + /// and meanings of levels (e.g. `lz4` switches to the slow high-compression mode starting from level 2), + /// so they use the generic `output_format_compression_level`. + auto compression_level_for_method = [&](CompressionMethod method) + { + if (method == CompressionMethod::Gzip || method == CompressionMethod::Zlib) + return static_cast(http_zlib_compression_level); + return static_cast(settings[Setting::output_format_compression_level]); + }; used_output.out_holder = std::make_shared( @@ -364,7 +374,7 @@ void HTTPHandler::processQuery( used_output.wrap_compressed_holder = wrapWriteBufferWithCompressionMethod( used_output.out.get(), http_response_compression_method, - static_cast(http_zlib_compression_level), + compression_level_for_method(http_response_compression_method), 0, DBMS_DEFAULT_BUFFER_SIZE, nullptr, diff --git a/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference new file mode 100644 index 0000000000000..a01d276d82828 --- /dev/null +++ b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.reference @@ -0,0 +1,6 @@ +gzip http_zlib_compression_level=9: 200 100000 4999950000 +xz http_zlib_compression_level=12: 200 100000 4999950000 +zstd http_zlib_compression_level=12: 200 100000 4999950000 +lz4 http_zlib_compression_level=12: 200 100000 4999950000 +zstd ignores http_zlib_compression_level +zstd uses output_format_compression_level diff --git a/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh new file mode 100755 index 0000000000000..9ddb0896adcd2 --- /dev/null +++ b/tests/queries/0_stateless/05398_http_zlib_compression_level_only_zlib.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `http_zlib_compression_level` applies only to the zlib-based methods (`gzip`, `deflate`). +# Other methods use `output_format_compression_level`. + +QUERY="SELECT+number+FROM+numbers(100000)" +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_response" + +function fetch() +{ + local encoding=$1 params=$2 extension=$3 + local code + code=$(${CLICKHOUSE_CURL} -sS -o "${FILE}.${extension}" -w '%{http_code}' -H "Accept-Encoding: ${encoding}" \ + "${CLICKHOUSE_URL}&enable_http_compression=1&${params}&query=${QUERY}") + echo -n "${encoding} ${params}: ${code} " + ${CLICKHOUSE_LOCAL} -q "SELECT count(), sum(toUInt64(line)) FROM file('${FILE}.${extension}', LineAsString)" +} + +function size() +{ + ${CLICKHOUSE_CURL} -sS -H "Accept-Encoding: $1" "${CLICKHOUSE_URL}&enable_http_compression=1&$2&query=${QUERY}" | wc -c +} + +fetch gzip "http_zlib_compression_level=9" gz +# Level 12 is valid for zlib with `libdeflate`, but not for `xz` (presets 0..9); it must not be passed to `xz`. +fetch xz "http_zlib_compression_level=12" xz +fetch zstd "http_zlib_compression_level=12" zst +fetch lz4 "http_zlib_compression_level=12" lz4 + +# `output_format_compression_level` controls the level of `zstd`, and `http_zlib_compression_level` does not. +[[ $(size zstd "http_zlib_compression_level=1") -eq $(size zstd "http_zlib_compression_level=9") ]] && echo "zstd ignores http_zlib_compression_level" +[[ $(size zstd "output_format_compression_level=1") -gt $(size zstd "output_format_compression_level=19") ]] && echo "zstd uses output_format_compression_level" + +rm -f "${FILE}".* From 1b3c520f0668b1111fefabf3daeae2119f7674f6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 03:52:01 +0000 Subject: [PATCH 09/46] Backport #124905 to 26.3: Reject CR/LF in outgoing HTTP request lines and headers --- base/poco/Net/src/HTTPRequest.cpp | 24 ++++++++++++++ src/Backups/BackupIO_S3.cpp | 9 ++++++ src/Core/YTsaurus/YTsaurusClient.cpp | 15 ++++++++- src/Core/YTsaurus/YTsaurusClient.h | 3 ++ src/Databases/DataLake/PaimonRestCatalog.cpp | 3 ++ src/Databases/DataLake/RestCatalog.cpp | 3 ++ .../ObjectStorages/S3/diskSettings.cpp | 11 +++++++ src/IO/GCPOAuth.cpp | 7 +++++ src/IO/GCPOAuth.h | 3 ++ src/IO/S3/PocoHTTPClient.cpp | 5 ++- src/IO/S3/Requests.cpp | 17 ++++++++++ src/IO/S3/Requests.h | 3 ++ src/IO/S3/copyS3File.cpp | 6 ++-- src/IO/WriteBufferFromS3.cpp | 2 +- src/Storages/StorageURL.cpp | 14 ++++----- src/Storages/YTsaurus/StorageYTsaurus.cpp | 9 +++++- .../05346_ytsaurus_header_injection.reference | 2 ++ .../05346_ytsaurus_header_injection.sh | 28 +++++++++++++++++ ...05347_http_request_crlf_rejected.reference | 3 ++ .../05347_http_request_crlf_rejected.sh | 31 +++++++++++++++++++ .../05348_url_tracestate_crlf.reference | 2 ++ .../0_stateless/05348_url_tracestate_crlf.sh | 17 ++++++++++ 22 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/05346_ytsaurus_header_injection.reference create mode 100755 tests/queries/0_stateless/05346_ytsaurus_header_injection.sh create mode 100644 tests/queries/0_stateless/05347_http_request_crlf_rejected.reference create mode 100755 tests/queries/0_stateless/05347_http_request_crlf_rejected.sh create mode 100644 tests/queries/0_stateless/05348_url_tracestate_crlf.reference create mode 100755 tests/queries/0_stateless/05348_url_tracestate_crlf.sh diff --git a/base/poco/Net/src/HTTPRequest.cpp b/base/poco/Net/src/HTTPRequest.cpp index 52b3cf2f714df..297b980d0b849 100644 --- a/base/poco/Net/src/HTTPRequest.cpp +++ b/base/poco/Net/src/HTTPRequest.cpp @@ -18,6 +18,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/Ascii.h" #include "Poco/String.h" +#include "Poco/Exception.h" using Poco::NumberFormatter; @@ -190,8 +191,31 @@ void HTTPRequest::setProxyCredentials(const std::string& scheme, const std::stri } +namespace +{ + bool hasLineBreak(const std::string& value) + { + return value.find_first_of("\r\n") != std::string::npos; + } + + /// CR or LF in the request line or in a header would let the value inject headers or a whole second request. + void checkNoLineBreaks(const HTTPRequest& request) + { + if (hasLineBreak(request.getMethod()) || hasLineBreak(request.getURI()) || hasLineBreak(request.getVersion())) + throw InvalidArgumentException("HTTP request line contains CR or LF"); + for (const auto& header : request) + { + if (hasLineBreak(header.first) || hasLineBreak(header.second)) + throw InvalidArgumentException("HTTP request header contains CR or LF", header.first); + } + } +} + + void HTTPRequest::write(std::ostream& ostr) const { + checkNoLineBreaks(*this); + ostr << _method << " " << _uri << " " << getVersion() << "\r\n"; HTTPMessage::write(ostr); ostr << "\r\n"; diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 78834e6d2d84f..a7ffc49efa602 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -3,6 +3,7 @@ #if USE_AWS_S3 #include #include +#include #include #include #include @@ -22,6 +23,7 @@ #include #include +#include namespace fs = std::filesystem; @@ -188,6 +190,13 @@ class S3BackupClientCreator .is_s3express_bucket = S3::isS3ExpressEndpoint(s3_uri.endpoint), }; + /// `gcp_oauth` adds an `Authorization` header to every request, so it must obey the same filter. + if (boost::iequals(client_configuration.http_client, "gcp_oauth")) + { + HTTPHeaderEntries gcp_oauth_headers{{"Authorization", "Bearer"}}; + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(gcp_oauth_headers); + } + return S3::ClientFactory::instance().create( client_configuration, client_settings, diff --git a/src/Core/YTsaurus/YTsaurusClient.cpp b/src/Core/YTsaurus/YTsaurusClient.cpp index 649e6057f0bb0..a37ce66692c3a 100644 --- a/src/Core/YTsaurus/YTsaurusClient.cpp +++ b/src/Core/YTsaurus/YTsaurusClient.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -37,9 +38,21 @@ namespace Setting extern const SettingsUInt64 max_http_get_redirects; } +static String authorizationHeaderValue(const String & oauth_token) +{ + return fmt::format("OAuth {}", oauth_token); +} + YTsaurusClient::YTsaurusClient(ContextPtr context_, const ConnectionInfo & connection_info_) : context(context_), connection_info(connection_info_), log(getLogger("YTsaurusClient")) { + checkHeaders(context, connection_info.oauth_token); +} + +void YTsaurusClient::checkHeaders(const ContextPtr & context, const String & oauth_token) +{ + HTTPHeaderEntries headers{{"Authorization", authorizationHeaderValue(oauth_token)}}; + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); } YTsaurusClient::YTsaurusClient(const YTsaurusClient & other) @@ -215,7 +228,7 @@ ReadBufferPtr YTsaurusClient::createQueryRWBuffer(const URI& uri, const ReadWrit /// Always use json format for input and output. {"Accept", "application/json"}, {"Content-Type", "application/json"}, - {"Authorization", fmt::format("OAuth {}", connection_info.oauth_token)}, + {"Authorization", authorizationHeaderValue(connection_info.oauth_token)}, {"X-YT-Header-Format", "yson"}, {"X-YT-Output-Format", fmt::format("{}json", output_params)}, }; diff --git a/src/Core/YTsaurus/YTsaurusClient.h b/src/Core/YTsaurus/YTsaurusClient.h index 1401e08df178b..de0d67cab368b 100644 --- a/src/Core/YTsaurus/YTsaurusClient.h +++ b/src/Core/YTsaurus/YTsaurusClient.h @@ -55,6 +55,9 @@ class YTsaurusClient : private boost::noncopyable explicit YTsaurusClient(ContextPtr context_, const ConnectionInfo & connection_info_); + /// Validates the OAuth token sent in the `Authorization` header with `HTTPHeaderFilter`. + static void checkHeaders(const ContextPtr & context, const String & oauth_token); + YTsaurusClient(const YTsaurusClient & other); const ConnectionInfo & getConnectionInfo() { return connection_info; } diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index c53859ee9f406..589c4cb8245c4 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -279,6 +280,8 @@ DB::ReadWriteBufferFromHTTPPtr PaimonRestCatalog::createReadBuffer( } DB::HTTPHeaderEntries request_headers(headers); createAuthHeaders(request_headers, endpoint, query_parameters_map, method); + /// The `dlf` provider adds headers built from the database settings (`dlf_access_key_id`, `region`). + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(request_headers); DB::WriteBufferFromOwnString headers_string; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 78e0fff6b1625..45f6fe8596950 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -582,6 +582,8 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( { auto result_headers = getAuthHeaders(update_token); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); + /// The auth headers may carry a token from the OAuth server or values from the database settings. + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(result_headers); return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -984,6 +986,7 @@ void RestCatalog::sendRequest(const String & endpoint, Poco::JSON::Object::Ptr r headers.emplace_back("Content-Type", "application/json"); const auto & context = getContext(); + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; if (!body_str.empty()) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp index 2b721c959db23..193734f86a2db 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp @@ -4,6 +4,7 @@ #if USE_AWS_S3 #include +#include #include #include #include @@ -22,6 +23,8 @@ #include #include +#include + #include #include #include @@ -225,6 +228,14 @@ getClient(const S3::URI & url, const S3Settings & settings, ContextPtr context, session_token = s3_updated_credentials->getSessionToken(); } } + + /// `gcp_oauth` adds an `Authorization` header to every request, so it must obey the same filter. + if (boost::iequals(client_configuration.http_client, "gcp_oauth")) + { + HTTPHeaderEntries gcp_oauth_headers{{"Authorization", "Bearer"}}; + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(gcp_oauth_headers); + } + return S3::ClientFactory::instance().create( client_configuration, client_settings, diff --git a/src/IO/GCPOAuth.cpp b/src/IO/GCPOAuth.cpp index 8d9b9eb7525bd..0938b68b5e1e4 100644 --- a/src/IO/GCPOAuth.cpp +++ b/src/IO/GCPOAuth.cpp @@ -102,10 +102,17 @@ GCPOAuthToken fetchGCPOAuthToken( GCPOAuthToken result; result.access_token = object->getValue("access_token"); + checkGCPOAuthAccessToken(result.access_token); if (object->has("expires_in")) result.expires_in = object->getValue("expires_in"); return result; } +void checkGCPOAuthAccessToken(const std::string & access_token) +{ + if (access_token.find_first_of("\r\n") != std::string::npos) + throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "GCP access token contains CR or LF"); +} + } diff --git a/src/IO/GCPOAuth.h b/src/IO/GCPOAuth.h index ebb0214a57c4f..faa31d75e930c 100644 --- a/src/IO/GCPOAuth.h +++ b/src/IO/GCPOAuth.h @@ -14,6 +14,9 @@ struct GCPOAuthToken Int64 expires_in = 3600; /// seconds until expiry as reported by the token endpoint }; +/// Throws if the access token from a token endpoint cannot be sent in an `Authorization` header. +void checkGCPOAuthAccessToken(const std::string & access_token); + /// Exchange a Google OAuth2 refresh token for an access token by POSTing to /// https://oauth2.googleapis.com/token. /// diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 8321ed96f96d9..d6178e244652d 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -883,9 +883,12 @@ PocoHTTPClientGCPOAuth::BearerToken PocoHTTPClientGCPOAuth::requestBearerToken() throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Unexpected structure of response. Expected Bearer token, got {}", token_type); + auto token = object->getValue("access_token"); + checkGCPOAuthAccessToken(token); + return { - .token = object->getValue("access_token"), + .token = std::move(token), .is_valid_to = std::chrono::system_clock::now() + std::chrono::seconds(object->getValue("expires_in")) }; } diff --git a/src/IO/S3/Requests.cpp b/src/IO/S3/Requests.cpp index ceff9057777b1..b329c051c1307 100644 --- a/src/IO/S3/Requests.cpp +++ b/src/IO/S3/Requests.cpp @@ -2,6 +2,7 @@ #if USE_AWS_S3 +#include #include #include #include @@ -9,9 +10,25 @@ #include #include +namespace DB +{ +namespace ErrorCodes +{ + extern const int INVALID_SETTING_VALUE; +} +} + namespace DB::S3 { +Model::StorageClass getStorageClassForName(const std::string & name) +{ + /// An unknown name is sent as is, so it must not break the header line. + if (name.find_first_of("\r\n") != std::string::npos) + throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting storage_class has invalid value: it contains CR or LF"); + return Model::StorageClassMapper::GetStorageClassForName(name); +} + Aws::Http::HeaderValueCollection CopyObjectRequest::GetRequestSpecificHeaders() const { auto headers = Model::CopyObjectRequest::GetRequestSpecificHeaders(); diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 7e2af17d9a5dc..fef30b836af76 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -61,6 +61,9 @@ inline void setChecksumAlgorithm(R & request) } }; +/// The `x-amz-storage-class` value for a storage class name; throws if the name contains CR or LF. +Model::StorageClass getStorageClassForName(const std::string & name); + template class ExtendedRequest : public BaseRequest { diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 9b31b37bf36d8..ad7c85bc72150 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -138,7 +138,7 @@ namespace const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) - request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); + request.SetStorageClass(S3::getStorageClassForName(storage_class_name)); client_ptr->setKMSHeaders(request); } @@ -471,7 +471,7 @@ namespace const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) - request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); + request.SetStorageClass(S3::getStorageClassForName(storage_class_name)); /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); @@ -679,7 +679,7 @@ namespace const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) - request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); + request.SetStorageClass(S3::getStorageClassForName(storage_class_name)); /// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840 request.SetContentType("binary/octet-stream"); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 4c131f24a1fa7..67b8bb53b6114 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -719,7 +719,7 @@ S3::PutObjectRequest WriteBufferFromS3::getPutRequest(PartData & data) if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); if (!request_settings[S3RequestSetting::storage_class_name].value.empty()) - req.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(request_settings[S3RequestSetting::storage_class_name])); + req.SetStorageClass(S3::getStorageClassForName(request_settings[S3RequestSetting::storage_class_name])); if (!write_settings.object_storage_write_if_none_match.empty()) req.SetIfNoneMatch(write_settings.object_storage_write_if_none_match); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 9f2fb9d55c3f1..2a15772c06905 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -221,7 +221,7 @@ IStorageURLBase::IStorageURLBase( namespace { - HTTPHeaderEntries getHeaders(const HTTPHeaderEntries & headers_) + HTTPHeaderEntries getHeaders(const HTTPHeaderEntries & headers_, const ContextPtr & context) { HTTPHeaderEntries headers(headers_.begin(), headers_.end()); @@ -229,12 +229,12 @@ namespace const auto & current_trace_context = OpenTelemetry::CurrentContext(); if (current_trace_context.isTraceEnabled()) { - headers.emplace_back("traceparent", current_trace_context.composeTraceparentHeader()); - + /// The user headers are checked in the `StorageURL` constructor; `tracestate` comes from the client as is. + HTTPHeaderEntries trace_headers{{"traceparent", current_trace_context.composeTraceparentHeader()}}; if (!current_trace_context.tracestate.empty()) - { - headers.emplace_back("tracestate", current_trace_context.tracestate); - } + trace_headers.emplace_back("tracestate", current_trace_context.tracestate); + context->getHTTPHeaderFilter().checkAndNormalizeHeaders(trace_headers); + headers.insert(headers.end(), trace_headers.begin(), trace_headers.end()); } return headers; @@ -347,7 +347,7 @@ StorageURLSource::StorageURLSource( , format_settings(format_settings_) , parser_shared_resources(std::move(parser_shared_resources_)) , format_filter_info(std::move(format_filter_info_)) - , headers(getHeaders(headers_)) + , headers(getHeaders(headers_, context_)) , need_only_count(need_only_count_) , hive_partition_columns_to_read_from_file_path(info.hive_partition_columns_to_read_from_file_path) { diff --git a/src/Storages/YTsaurus/StorageYTsaurus.cpp b/src/Storages/YTsaurus/StorageYTsaurus.cpp index eaf4566b9c75c..9b93a5c9f9c26 100644 --- a/src/Storages/YTsaurus/StorageYTsaurus.cpp +++ b/src/Storages/YTsaurus/StorageYTsaurus.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -147,9 +148,15 @@ void registerStorageYTsaurus(StorageFactory & factory) if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_ytsaurus_table_engine]) throw Exception(ErrorCodes::UNKNOWN_STORAGE, "Table engine YTsaurus is experimental. " "Set `allow_experimental_ytsaurus_table_engine` setting to enable it"); + auto configuration = StorageYTsaurus::getConfiguration( + args.engine_args, YTsaurusSettings::createFromQuery(*args.storage_def), args.getLocalContext(), &args.table_id); + /// Skipped for a stored definition so that a forbidden header does not block server startup; + /// the client still checks it. + if (isFreshTableDefinition(args.mode, args.query.attach_short_syntax) && !args.getLocalContext()->isRecoveryFromStoredMetadata()) + YTsaurusClient::checkHeaders(args.getLocalContext(), configuration.oauth_token); return std::make_shared( args.table_id, - StorageYTsaurus::getConfiguration(args.engine_args, YTsaurusSettings::createFromQuery(*args.storage_def), args.getLocalContext(), &args.table_id), + std::move(configuration), args.columns, args.constraints, args.comment); diff --git a/tests/queries/0_stateless/05346_ytsaurus_header_injection.reference b/tests/queries/0_stateless/05346_ytsaurus_header_injection.reference new file mode 100644 index 0000000000000..c3d4f625310cc --- /dev/null +++ b/tests/queries/0_stateless/05346_ytsaurus_header_injection.reference @@ -0,0 +1,2 @@ +0 +OK diff --git a/tests/queries/0_stateless/05346_ytsaurus_header_injection.sh b/tests/queries/0_stateless/05346_ytsaurus_header_injection.sh new file mode 100755 index 0000000000000..374df199f198f --- /dev/null +++ b/tests/queries/0_stateless/05346_ytsaurus_header_injection.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The OAuth token is sent in the Authorization header, so it must not contain CR or LF. +# The proxy URL points to a closed port, so an unchecked token fails with a network error instead. +# A full-definition ATTACH is user input like CREATE; Atomic and Replicated databases need a UUID for it. +UUID_CLAUSE="" +if [ "$($CLICKHOUSE_CLIENT -q "SELECT engine FROM system.databases WHERE name = currentDatabase()")" != "Ordinary" ]; then + UUID_CLAUSE="UUID '$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()")'" +fi + +$CLICKHOUSE_CLIENT -m <&1 \ + | grep -o -m1 "HTTP request header contains CR or LF" + +# An unknown storage class is sent as is in `x-amz-storage-class` when `s3_validate_request_settings = 0`, +# so CR or LF in it must be rejected before the request is made. +$CLICKHOUSE_CLIENT -q " + INSERT INTO FUNCTION s3('http://localhost:${CLICKHOUSE_PORT_HTTP}/bucket/${CLICKHOUSE_DATABASE}.csv', 'key', 'secret', 'CSV', 'x UInt8', + storage_class_name = 'STANDARD\r\nX-Injected: 1') + SETTINGS s3_validate_request_settings = 0, s3_truncate_on_insert = 1 + SELECT 1" 2>&1 | grep -o -m1 "Setting storage_class has invalid value: it contains CR or LF" + +$CLICKHOUSE_CLIENT -m -q " + DROP TABLE IF EXISTS t_storage_class; + CREATE TABLE t_storage_class (x UInt8) ENGINE = MergeTree ORDER BY x; + INSERT INTO t_storage_class SELECT 1; +" +$CLICKHOUSE_CLIENT -q " + BACKUP TABLE t_storage_class TO S3('http://localhost:${CLICKHOUSE_PORT_HTTP}/bucket/${CLICKHOUSE_DATABASE}_backup', 'key', 'secret') + SETTINGS s3_storage_class = 'STANDARD\r\nX-Injected: 1'" 2>&1 | grep -o -m1 "Setting storage_class has invalid value: it contains CR or LF" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_storage_class" diff --git a/tests/queries/0_stateless/05348_url_tracestate_crlf.reference b/tests/queries/0_stateless/05348_url_tracestate_crlf.reference new file mode 100644 index 0000000000000..0dd911d458ede --- /dev/null +++ b/tests/queries/0_stateless/05348_url_tracestate_crlf.reference @@ -0,0 +1,2 @@ +HTTP header "tracestate" has invalid character +1 diff --git a/tests/queries/0_stateless/05348_url_tracestate_crlf.sh b/tests/queries/0_stateless/05348_url_tracestate_crlf.sh new file mode 100755 index 0000000000000..c05c60ab14cee --- /dev/null +++ b/tests/queries/0_stateless/05348_url_tracestate_crlf.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The client's OpenTelemetry `tracestate` is forwarded by `url` as an HTTP header, so CR or LF in it must be rejected. +$CLICKHOUSE_CLIENT \ + --opentelemetry-traceparent "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" \ + --opentelemetry-tracestate $'a=b\r\nX-Injected: 1' \ + -q "SELECT * FROM url('http://localhost:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', 'TSV', 'x UInt8')" 2>&1 \ + | grep -o -m1 'HTTP header "tracestate" has invalid character' + +$CLICKHOUSE_CLIENT \ + --opentelemetry-traceparent "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" \ + --opentelemetry-tracestate 'a=b' \ + -q "SELECT * FROM url('http://localhost:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', 'TSV', 'x UInt8')" From 0fcdf8af90c5f5df23eda4e3e49a90dcafb46947 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 13:45:03 +0000 Subject: [PATCH 10/46] Update autogenerated version to 26.3.54.6 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 4f96015e94281..2a05c6bdfe58d 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54561) +SET(VERSION_REVISION 54562) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 54) -SET(VERSION_GITHASH d8af9de0b15bce32bc1e9e4b48d3e234f2cff5cd) -SET(VERSION_DESCRIBE v26.3.54.1-lts) -SET(VERSION_STRING 26.3.54.1) +SET(VERSION_PATCH 55) +SET(VERSION_GITHASH 9f3fea88bbeea3363fb769d5a4548f294414d0f9) +SET(VERSION_DESCRIBE v26.3.55.1-lts) +SET(VERSION_STRING 26.3.55.1) # end of autochange From 2dac2e7f965042660405fd5bdf6430e2f85d57c8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 17 Oct 2026 15:46:10 +0000 Subject: [PATCH 11/46] Backport #118866 to 26.3: Require SELECT on the source when CREATE TABLE AS inherits credentials --- src/Interpreters/InterpreterCreateQuery.cpp | 44 +++++++++++ ..._external_engine_requires_select.reference | 23 ++++++ ...eate_as_external_engine_requires_select.sh | 73 +++++++++++++++++++ ..._settings_secret_requires_select.reference | 6 ++ ..._engine_settings_secret_requires_select.sh | 43 +++++++++++ ..._on_cluster_inherits_credentials.reference | 17 +++++ ...eate_as_on_cluster_inherits_credentials.sh | 69 ++++++++++++++++++ ..._series_inner_engine_credentials.reference | 6 ++ ...as_time_series_inner_engine_credentials.sh | 45 ++++++++++++ ...te_as_secret_key_requires_select.reference | 8 ++ ...32_create_as_secret_key_requires_select.sh | 44 +++++++++++ 11 files changed, 378 insertions(+) create mode 100644 tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference create mode 100755 tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh create mode 100644 tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference create mode 100755 tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh create mode 100644 tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference create mode 100755 tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh create mode 100644 tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference create mode 100755 tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh create mode 100644 tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference create mode 100755 tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 1e55fed06c5de..715a96ebc0b74 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1192,6 +1192,8 @@ namespace { *ptr = nullptr; }); + /// `children` keeps the old nodes, and `hasSecretParts` reads them + storage.children.clear(); auto engine_ast = make_intrusive(); engine_ast->name = "Null"; @@ -1341,12 +1343,23 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const String as_database_name = getContext()->resolveDatabase(create.as_database); String as_table_name = create.as_table; + /// Check SHOW COLUMNS first, or the error could reveal that an inaccessible table holds credentials. + getContext()->checkAccess(AccessType::SHOW_COLUMNS, as_database_name, as_table_name); + ASTPtr as_create_ptr = DatabaseCatalog::instance().getDatabase(as_database_name)->getCreateTableQuery(as_table_name, getContext()); const auto & as_create = as_create_ptr->as(); const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name); + /// SHOW CREATE TABLE masks credentials, so inheriting the definition could expose data to a user + /// who cannot SELECT from the source. SHOW COLUMNS already authorizes the rest of the definition. + auto check_access_to_inherited_definition = [&](const IAST & definition) + { + if (definition.hasSecretParts()) + getContext()->checkAccess(AccessType::SELECT, as_database_name, as_table_name); + }; + if (as_create.is_ordinary_view) throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a View", qualified_name); @@ -1371,6 +1384,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const } else if (as_create.as_table_function) { + check_access_to_inherited_definition(*as_create.as_table_function); create.set(create.as_table_function, as_create.as_table_function->ptr()); return; } @@ -1378,11 +1392,19 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { storage_def = boost::static_pointer_cast(as_create.storage->ptr()); create.is_time_series_table = as_create.is_time_series_table; + + if (as_create.is_time_series_table && as_create.targets) + for (const auto & target : as_create.targets->targets) + if (target.inner_engine && !create.getTargetInnerEngine(target.kind)) + check_access_to_inherited_definition(*target.inner_engine); } else { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set engine, it's a bug."); } + + if (storage_def) + check_access_to_inherited_definition(*storage_def); } if (!storage_def) @@ -2498,7 +2520,29 @@ BlockIO InterpreterCreateQuery::execute() auto on_cluster_version = getContext()->getSettingsRef()[Setting::distributed_ddl_entry_format_version]; if (is_create_database || on_cluster_version < DDLLogEntry::NORMALIZE_CREATE_ON_INITIATOR_VERSION) + { + /// This branch returns before setEngine, and the worker processes AS src without the user's access. + if (!create.as_table.empty()) + { + create.as_database = getContext()->resolveDatabase(create.as_database); + + auto preflight_context = Context::createCopy(getContext()); + if (on_cluster_version == DDLLogEntry::OLDEST_VERSION) + { + preflight_context->setSetting("restore_replace_external_engines_to_null", false); + preflight_context->setSetting("restore_replace_external_table_functions_to_null", false); + } + + ASTPtr inherited_query = query_ptr->clone(); + auto & inherited = inherited_query->as(); + InterpreterCreateQuery(inherited_query, preflight_context).setEngine(inherited); + + if (inherited.storage && inherited.storage->engine) + getContext()->checkAccess(AccessType::TABLE_ENGINE, inherited.storage->engine->name); + } + return executeQueryOnCluster(create); + } } getContext()->checkAccess(getRequiredAccess()); diff --git a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference new file mode 100644 index 0000000000000..f0edc2dad6cfa --- /dev/null +++ b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference @@ -0,0 +1,23 @@ +with SHOW COLUMNS only: +-- copy_of_local_src: +MergeTree +-- copy_of_url_src: +necessary to have the grant SELECT ON db.url_src +-- copy_of_function_src: +necessary to have the grant SELECT ON db.function_src +-- copy_of_url_src_no_password: +URL +-- copy_of_function_src_no_password: +Proxy +-- null_copy_of_url_src: +Null +after GRANT SELECT: +-- copy_of_url_src: +URL +-- copy_of_function_src: +Proxy +without SHOW COLUMNS: +-- local_src: +necessary to have the grant SHOW COLUMNS ON db.local_src +-- url_src: +necessary to have the grant SHOW COLUMNS ON db.url_src diff --git a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh new file mode 100755 index 0000000000000..22f7199b241af --- /dev/null +++ b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# CREATE TABLE x AS y copies the engine of y with its credentials, which SHOW CREATE TABLE masks. +# That copy needs SELECT on y. Any other definition needs only SHOW COLUMNS. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +blind="blind_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +# the server does not contact the URLs. it only copies the definitions +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}, ${blind}; + CREATE USER ${user}, ${blind}; + GRANT CREATE TABLE ON ${db}.* TO ${user}, ${blind}; + GRANT TABLE ENGINE ON MergeTree, TABLE ENGINE ON Null TO ${user}, ${blind}; + GRANT URL ON *.* TO ${user}, ${blind}; + + CREATE TABLE ${db}.local_src (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE ${db}.url_src (id UInt64) ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.url_src_no_password (id UInt64) ENGINE = URL('http://127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.function_src (id UInt64) AS url('http://user:password@127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.function_src_no_password (id UInt64) AS url('http://127.0.0.1:1/', 'CSV'); + + GRANT SHOW COLUMNS ON ${db}.local_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.url_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.url_src_no_password TO ${user}; + GRANT SHOW COLUMNS ON ${db}.function_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.function_src_no_password TO ${user}; +" + +# prints the missing privilege, or the engine of the copy. more arguments go to the client +function try_copy() +{ + local name=$1 source=$2 + shift 2 + echo "-- ${name}:" + ${CLICKHOUSE_CLIENT} --user "${user}" "${@}" -q "CREATE TABLE ${db}.${name} AS ${db}.${source}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = '${name}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_of_local_src local_src +try_copy copy_of_url_src url_src +try_copy copy_of_function_src function_src +# these two hold nothing masked, so the server copies them as before +try_copy copy_of_url_src_no_password url_src_no_password +try_copy copy_of_function_src_no_password function_src_no_password +# Null replaces the engine here, so the copy inherits nothing masked +try_copy null_copy_of_url_src url_src --restore_replace_external_engines_to_null 1 + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q " + GRANT SELECT ON ${db}.url_src TO ${user}; + GRANT SELECT ON ${db}.function_src TO ${user}; +" +try_copy copy_of_url_src url_src +try_copy copy_of_function_src function_src + +# a user who cannot see the source gets the same error for both tables +echo "without SHOW COLUMNS:" +for src in local_src url_src +do + echo "-- ${src}:" + ${CLICKHOUSE_CLIENT} --user "${blind}" -q "CREATE TABLE ${db}.blind_copy AS ${db}.${src}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" +done + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}, ${blind}" diff --git a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference new file mode 100644 index 0000000000000..5a636ec83f4c1 --- /dev/null +++ b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference @@ -0,0 +1,6 @@ +with SHOW COLUMNS only: +necessary to have the grant SELECT ON db.kafka_src +with a password in this query: +Kafka +after GRANT SELECT: +Kafka diff --git a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh new file mode 100755 index 0000000000000..e810ad7954ede --- /dev/null +++ b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test does not build the Kafka engine + +# the engine settings can hold credentials, not only the engine arguments. SHOW CREATE TABLE masks +# kafka_sasl_password too, so a copy of it also needs SELECT. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +# the server does not contact the broker. it only copies the definition +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT KAFKA ON *.* TO ${user}; + + CREATE TABLE ${db}.kafka_src (id UInt64) ENGINE = Kafka + SETTINGS kafka_broker_list = 'localhost:9092', kafka_topic_list = 'topic', + kafka_group_name = 'group', kafka_format = 'CSV', kafka_sasl_password = 'secret'; + + GRANT SHOW COLUMNS ON ${db}.kafka_src TO ${user}; +" + +echo "with SHOW COLUMNS only:" +${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.copy_of_kafka_src AS ${db}.kafka_src" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + +# the copy does not inherit a setting that this query gives, so it needs nothing more +echo "with a password in this query:" +${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.own_password AS ${db}.kafka_src SETTINGS kafka_sasl_password = 'own'" +${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = 'own_password'" + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.kafka_src TO ${user}" +${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.copy_of_kafka_src AS ${db}.kafka_src" +${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = 'copy_of_kafka_src'" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference new file mode 100644 index 0000000000000..eae6826a3c065 --- /dev/null +++ b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.reference @@ -0,0 +1,17 @@ +with SHOW COLUMNS only: +-- copy_legacy: +necessary to have the grant SELECT ON db.url_src +-- copy_current: +necessary to have the grant SELECT ON db.url_src +-- copy_legacy_unqualified: +necessary to have the grant SELECT ON db.url_src +-- copy_legacy_plain: +MergeTree +after GRANT SELECT: +-- copy_legacy: +URL +-- copy_legacy_unqualified: +URL +with SELECT but without the grant for the engine: +-- copy_no_url: +necessary to have the grant TABLE ENGINE ON URL diff --git a/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh new file mode 100755 index 0000000000000..a398557e009fb --- /dev/null +++ b/tests/queries/0_stateless/05238_create_as_on_cluster_inherits_credentials.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database +# Tag no-replicated-database: a Replicated database does not allow ON CLUSTER + +# an entry format before NORMALIZE_CREATE_ON_INITIATOR_VERSION holds the query as written, and the +# worker builds AS src with no user. the initiator must check the source itself. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +no_url="no_url_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT CLUSTER ON *.* TO ${user}; + GRANT URL ON *.* TO ${user}; + GRANT TABLE ENGINE ON MergeTree TO ${user}; + + DROP USER IF EXISTS ${no_url}; + CREATE USER ${no_url}; + GRANT CREATE TABLE ON ${db}.* TO ${no_url}; + GRANT CLUSTER ON *.* TO ${no_url}; + + CREATE TABLE ${db}.url_src (id UInt64) ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + CREATE TABLE ${db}.plain_src (id UInt64) ENGINE = MergeTree ORDER BY id; + + GRANT SHOW COLUMNS ON ${db}.url_src TO ${user}; + GRANT SHOW COLUMNS ON ${db}.plain_src TO ${user}; + GRANT SELECT ON ${db}.url_src TO ${no_url}; +" + +legacy=(--user "${user}" --distributed_ddl_output_mode throw --distributed_ddl_entry_format_version 2) +current=(--user "${user}" --distributed_ddl_output_mode throw) + +# prints the missing privilege, or the engine of the copy. each case uses its own name +function try_copy() +{ + local name=$1 source=$2 + shift 2 + echo "-- ${name}:" + ${CLICKHOUSE_CLIENT} "${@}" -q "CREATE TABLE ${db}.${name} ON CLUSTER test_shard_localhost AS ${source}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ([A-Za-z]+|${db}\.[a-z_]+)" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = '${name}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_legacy "${db}.url_src" "${legacy[@]}" +try_copy copy_current "${db}.url_src" "${current[@]}" +# the server checks and reads a source without a database name in the database of this query +try_copy copy_legacy_unqualified url_src "${legacy[@]}" +# this one holds nothing masked, so the copy needs no SELECT +try_copy copy_legacy_plain "${db}.plain_src" "${legacy[@]}" + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.url_src TO ${user}" +try_copy copy_legacy "${db}.url_src" "${legacy[@]}" +try_copy copy_legacy_unqualified url_src "${legacy[@]}" + +# the engine comes with the source, so the copy needs the grant for the engine too +echo "with SELECT but without the grant for the engine:" +try_copy copy_no_url "${db}.url_src" --user "${no_url}" --distributed_ddl_output_mode throw \ + --distributed_ddl_entry_format_version 2 + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}, ${no_url}" diff --git a/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference new file mode 100644 index 0000000000000..a24472ca367f7 --- /dev/null +++ b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.reference @@ -0,0 +1,6 @@ +with SHOW COLUMNS only: +-- copy_of_ts_src: +necessary to have the grant SELECT ON db.ts_src +after GRANT SELECT: +-- copy_of_ts_src: +TimeSeries diff --git a/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh new file mode 100755 index 0000000000000..e110640472135 --- /dev/null +++ b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# a TimeSeries source holds its credentials in the inner engines, which the outer engine does not +# show. the copy inherits them, so it needs SELECT on the source. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" +client=(${CLICKHOUSE_CLIENT} --allow_experimental_time_series_table 1) + +# the URL is never contacted. only the definition is copied +"${client[@]}" -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT URL ON *.* TO ${user}; + GRANT TABLE ENGINE ON TimeSeries, TABLE ENGINE ON MergeTree, TABLE ENGINE ON AggregatingMergeTree, + TABLE ENGINE ON ReplacingMergeTree, TABLE ENGINE ON URL TO ${user}; + + CREATE TABLE ${db}.ts_src ENGINE = TimeSeries + SAMPLES ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + + GRANT SHOW COLUMNS ON ${db}.ts_src TO ${user}; +" + +# prints the missing privilege, or the engine of the copy +function try_copy() +{ + echo "-- ${1}:" + "${client[@]}" --user "${user}" -q "CREATE TABLE ${db}.${1} AS ${db}.ts_src" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = '${1}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_of_ts_src + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.ts_src TO ${user}" +try_copy copy_of_ts_src + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference new file mode 100644 index 0000000000000..e4ba420be9779 --- /dev/null +++ b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.reference @@ -0,0 +1,8 @@ +with SHOW COLUMNS only: +-- copy_of_key_src: +necessary to have the grant SELECT ON db.key_src +-- own_key: +s +after GRANT SELECT: +-- copy_of_key_src: +encrypt(\'aes-128-ecb\', \'[HIDDEN]\') diff --git a/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh new file mode 100755 index 0000000000000..8deb698cdb9c9 --- /dev/null +++ b/tests/queries/0_stateless/05332_create_as_secret_key_requires_select.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test does not build the encryption functions + +# a key expression can hold credentials too, and SHOW CREATE TABLE masks them. the copy needs SELECT +# on the source, unless this query gives its own key. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q " + DROP USER IF EXISTS ${user}; + CREATE USER ${user}; + GRANT CREATE TABLE ON ${db}.* TO ${user}; + GRANT TABLE ENGINE ON MergeTree TO ${user}; + + CREATE TABLE ${db}.key_src (s String) ENGINE = MergeTree ORDER BY encrypt('aes-128-ecb', s, '0123456789abcdef'); + + GRANT SHOW COLUMNS ON ${db}.key_src TO ${user}; +" + +# prints the missing privilege, or the sorting key of the copy +function try_copy() +{ + local name=$1 clauses=$2 + echo "-- ${name}:" + ${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.${name} AS ${db}.key_src ${clauses}" 2>&1 \ + | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" + ${CLICKHOUSE_CLIENT} -q "SELECT sorting_key FROM system.tables WHERE database = '${db}' AND name = '${name}'" +} + +echo "with SHOW COLUMNS only:" +try_copy copy_of_key_src "" +try_copy own_key "ORDER BY s" + +echo "after GRANT SELECT:" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.key_src TO ${user}" +try_copy copy_of_key_src "" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" From 358652e581bd45cad5e2e439cf7b321a45034ad0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 18 Oct 2026 13:39:00 +0000 Subject: [PATCH 12/46] Update autogenerated version to 26.3.55.2 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2a05c6bdfe58d..8677193ae1330 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54562) +SET(VERSION_REVISION 54563) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 55) -SET(VERSION_GITHASH 9f3fea88bbeea3363fb769d5a4548f294414d0f9) -SET(VERSION_DESCRIBE v26.3.55.1-lts) -SET(VERSION_STRING 26.3.55.1) +SET(VERSION_PATCH 56) +SET(VERSION_GITHASH e029f7835cc3f7d6e22db137c01d75c30f855893) +SET(VERSION_DESCRIBE v26.3.56.1-lts) +SET(VERSION_STRING 26.3.56.1) # end of autochange From dbf90f8cb50fc6692b42d8ce66af3eadc2775b13 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 12:04:56 +0000 Subject: [PATCH 13/46] Backport #124075 to 26.3: Leave only loadable objects behind in `05316_named_collection_dictionary_override_access` --- ...d_collection_dictionary_override_access.sh | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh index 2039d59fd0bdb..9bbc91a03f3eb 100755 --- a/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh +++ b/tests/queries/0_stateless/05316_named_collection_dictionary_override_access.sh @@ -10,6 +10,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e nc="${CLICKHOUSE_TEST_UNIQUE_NAME}_dict" +nc_locked="${CLICKHOUSE_TEST_UNIQUE_NAME}_locked" nc_load="${CLICKHOUSE_TEST_UNIQUE_NAME}_load" nc_mysql="${CLICKHOUSE_TEST_UNIQUE_NAME}_mysql" nc_url="${CLICKHOUSE_TEST_UNIQUE_NAME}_url" @@ -17,7 +18,19 @@ user="${CLICKHOUSE_TEST_UNIQUE_NAME}_user" function cleanup() { + # A client killed inside attach_is_rejected leaves its table detached permanently, which DROP TABLE does not see. + for table in $(${CLICKHOUSE_CLIENT} --query "SELECT table FROM system.detached_tables + WHERE database = currentDatabase() AND table IN ('table_settings_override', 'table_alias_override')"); do + ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; + ALTER NAMED COLLECTION IF EXISTS $nc_mysql SET connection_pool_size = 2 OVERRIDABLE; + ALTER NAMED COLLECTION IF EXISTS $nc_url SET http_method = 'POST' OVERRIDABLE; + ATTACH TABLE $table; + " + done + # Neither ignore nor fuzz these drops: each object must be gone before the named collection it references. ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0, ignore_drop_queries_probability = 0; DROP DICTIONARY IF EXISTS dict_override; DROP DICTIONARY IF EXISTS dict_add_key; DROP DICTIONARY IF EXISTS dict_alias; @@ -29,6 +42,7 @@ function cleanup() DROP TABLE IF EXISTS dict_source_b; DROP USER IF EXISTS $user; DROP NAMED COLLECTION IF EXISTS $nc; + DROP NAMED COLLECTION IF EXISTS $nc_locked; DROP NAMED COLLECTION IF EXISTS $nc_load; DROP NAMED COLLECTION IF EXISTS $nc_mysql; DROP NAMED COLLECTION IF EXISTS $nc_url; @@ -46,6 +60,9 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " CREATE NAMED COLLECTION $nc AS host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a'; + CREATE NAMED COLLECTION $nc_locked AS + host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', + db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a' NOT OVERRIDABLE; CREATE NAMED COLLECTION $nc_load AS host = '127.0.0.1', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}', table = 'dict_source_a'; @@ -54,6 +71,7 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " GRANT SOURCES ON *.* TO $user; GRANT CREATE DICTIONARY, DROP DICTIONARY, dictGet ON ${CLICKHOUSE_DATABASE}.* TO $user; GRANT NAMED COLLECTION ON $nc TO $user; + GRANT NAMED COLLECTION ON $nc_locked TO $user; " echo 'Overriding a stored key in a dictionary source requires the secrets privilege' @@ -83,10 +101,11 @@ ${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " " echo 'A NOT OVERRIDABLE key cannot be overridden even with the secrets privilege' -${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc SET table = 'dict_source_a' NOT OVERRIDABLE" +# A separate collection: no stored dictionary overrides a key while it is locked, so a restart can load what the test leaves. +${CLICKHOUSE_CLIENT} --query "GRANT SHOW NAMED COLLECTIONS SECRETS ON $nc_locked TO $user" ${CLICKHOUSE_CLIENT} --user "$user" --multiquery --query " CREATE DICTIONARY dict_locked (id UInt64, value UInt64) PRIMARY KEY id - SOURCE(CLICKHOUSE(NAME $nc TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError BAD_ARGUMENTS } + SOURCE(CLICKHOUSE(NAME $nc_locked TABLE 'dict_source_b')) LAYOUT(FLAT()) LIFETIME(0); -- { serverError BAD_ARGUMENTS } " echo 'A dictionary created with an override by a privileged user loads in the background' @@ -102,7 +121,9 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " " echo 'A stored table whose override became NOT OVERRIDABLE is rejected when it is attached' +# The fuzzer stays off for all DDL of these tables: a __fuzz_N clone would stay attached while its key is locked. ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; CREATE NAMED COLLECTION $nc_mysql AS host = '127.0.0.1', port = 1, user = 'user', password = 'secret', database = 'database', table = 'table', connection_pool_size = 2; @@ -113,28 +134,30 @@ ${CLICKHOUSE_CLIENT} --multiquery --query " " # A table that overrides a key which is locked afterwards cannot be attached: the lock is checked on every load. +# The table stays detached permanently while its key is locked, so a server restart never loads it. function attach_is_rejected() { local table=$1 - local key=$2 - ${CLICKHOUSE_CLIENT} --query "DETACH TABLE $table" - if error=$(${CLICKHOUSE_CLIENT} --query "ATTACH TABLE $table" 2>&1); then + local collection=$2 + local key=$3 + local value=$4 + ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; + DETACH TABLE $table PERMANENTLY; + ALTER NAMED COLLECTION $collection SET $key = $value NOT OVERRIDABLE; + " + if error=$(${CLICKHOUSE_CLIENT} --multiquery --query "SET ast_fuzzer_any_query = 0; ATTACH TABLE $table" 2>&1); then echo "Expected the attach of $table to be rejected" exit 1 fi echo "$error" | grep -o "Override not allowed for '$key'" | head -1 echo "$error" | grep -o 'BAD_ARGUMENTS' | head -1 + ${CLICKHOUSE_CLIENT} --multiquery --query " + SET ast_fuzzer_any_query = 0; + ALTER NAMED COLLECTION $collection SET $key = $value OVERRIDABLE; + ATTACH TABLE $table; + " } -${CLICKHOUSE_CLIENT} --query "ALTER NAMED COLLECTION $nc_mysql SET connection_pool_size = 2 NOT OVERRIDABLE" -attach_is_rejected table_settings_override connection_pool_size -${CLICKHOUSE_CLIENT} --multiquery --query " - ALTER NAMED COLLECTION $nc_mysql SET connection_pool_size = 2 OVERRIDABLE; - ATTACH TABLE table_settings_override; - ALTER NAMED COLLECTION $nc_url SET http_method = 'POST' NOT OVERRIDABLE; -" -attach_is_rejected table_alias_override http_method -${CLICKHOUSE_CLIENT} --multiquery --query " - ALTER NAMED COLLECTION $nc_url SET http_method = 'POST' OVERRIDABLE; - ATTACH TABLE table_alias_override; -" +attach_is_rejected table_settings_override "$nc_mysql" connection_pool_size 2 +attach_is_rejected table_alias_override "$nc_url" http_method "'POST'" From 15856f6f6457452d2c8f4aad437dd75b748afa16 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 13:32:15 +0000 Subject: [PATCH 14/46] Backport #120407 to 26.3: [SECURITY] Mask credentials embedded in HTTP URIs --- src/Common/maskURIPassword.h | 18 +++ src/IO/HTTPCommon.cpp | 50 +++++++- src/IO/HTTPCommon.h | 19 ++- src/IO/ReadWriteBufferFromHTTP.cpp | 26 ++-- src/IO/WriteBufferFromHTTP.cpp | 6 +- src/Storages/StorageURL.cpp | 10 +- .../StorageSystemSchemaInferenceCache.cpp | 29 ++++- ...5212_http_uri_credentials_masked.reference | 9 ++ .../05212_http_uri_credentials_masked.sh | 115 ++++++++++++++++++ 9 files changed, 262 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/05212_http_uri_credentials_masked.reference create mode 100755 tests/queries/0_stateless/05212_http_uri_credentials_masked.sh diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index 551371c590f69..d368e482aa8c9 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -102,4 +102,22 @@ inline bool maskPresignedURLParameters(std::string & url) return true; } +/** Mask both credential carriers of a URL: the userinfo (`user:password@`) and the values of the + * presigned-URL query parameters. Returns whether anything was masked. + * + * This is the masking to use for a whole HTTP/S3 URL. Scanning for a presigned parameter is only + * sound on a value that is one whole URL (see `maskPresignedURLParameters`), so the argument must be + * one URL, not a URL embedded in a longer string. + */ +inline bool maskURICredentials(std::string & url) +{ + /// This release has no `maskURIUserinfo`; this is the regular expression it replaced. The userinfo + /// is taken greedily up to the last at-sign before the path, so a password that itself contains an + /// at-sign is masked whole. + static const re2::RE2 userinfo(R"(^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/?#]+@)"); + bool masked = RE2::Replace(&url, userinfo, "\\1[HIDDEN]@"); + masked |= maskPresignedURLParameters(url); + return masked; +} + } diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 4a5f46449eec3..af42b33980565 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -3,6 +3,9 @@ #include #include #include +#include + +#include #include "config.h" @@ -64,11 +67,36 @@ std::istream * receiveResponse( Poco::Net::HTTPClientSession & session, const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, const bool allow_redirects) { auto & istr = session.receiveResponse(response); - assertResponseIsOk(request.getURI(), response, istr, allow_redirects); + assertResponseIsOk(request.getURI(), response, istr, allow_redirects, requestCredentialSecrets(request)); return &istr; } -void assertResponseIsOk(const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, const bool allow_redirects) +Strings requestCredentialSecrets(const Poco::Net::HTTPRequest & request) +{ + if (!request.has("Authorization")) + return {}; + + /// `getCredentials` splits the scheme off the header value the way an HTTP peer does: the scheme + /// is matched case-insensitively, and a field value may begin with whitespace (RFC 9110, 5.5), + /// which `auth_header = 'Authorization: Bearer '` leaves in front of the scheme. + std::string scheme; + std::string auth_info; + request.getCredentials(scheme, auth_info); + Poco::trimInPlace(auth_info); + /// Only a bearer token is recoverable from the header, and it is a secret as a whole. A `Basic` + /// header holds base64 of `user:password` and is deliberately not decoded here: its value is + /// user input whenever it came from `headers()`, decoding throws on anything outside the base64 + /// alphabet, and the user name of a pair is an identifier rather than a credential. A password + /// is therefore not scrubbed from a response body; no peer is known to echo one back. + if (Poco::icompare(scheme, "Bearer") == 0 && !auth_info.empty()) + return {auth_info}; + + return {}; +} + +void assertResponseIsOk( + const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, const bool allow_redirects, + const Strings & body_secrets) { auto status = response.getStatus(); @@ -85,7 +113,7 @@ void assertResponseIsOk(const String & uri, Poco::Net::HTTPResponse & response, std::string body; Poco::StreamCopier::copyToString(istr, body); - throw HTTPException(code, uri, status, response.getReason(), body); + throw HTTPException(code, uri, status, response.getReason(), body, body_secrets); } } @@ -94,13 +122,25 @@ Exception HTTPException::makeExceptionMessage( const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus http_status, const std::string & reason, - const std::string & body) + const std::string & body, + const Strings & body_secrets) { + std::string masked_uri = uri; + maskURICredentials(masked_uri); + + /// The response body is remote content that can reflect the request's own credentials back, which + /// a server may echo verbatim (e.g. an auth error naming the user). Scrub those exact credential + /// strings, keeping the rest of the body - a useful diagnostic that does not carry a credential. + std::string masked_body = body; + for (const auto & secret : body_secrets) + if (!secret.empty()) + boost::replace_all(masked_body, secret, "[HIDDEN]"); + return Exception(code, "Received error from remote server {}. " "HTTP status code: {} '{}', " "body length: {} bytes, body: '{}'", - uri, static_cast(http_status), reason, body.length(), body); + masked_uri, static_cast(http_status), reason, body.length(), masked_body); } } diff --git a/src/IO/HTTPCommon.h b/src/IO/HTTPCommon.h index abf249cddff24..19c01aa7a8b9c 100644 --- a/src/IO/HTTPCommon.h +++ b/src/IO/HTTPCommon.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -28,9 +29,10 @@ class HTTPException : public Exception const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus http_status_, const std::string & reason, - const std::string & body + const std::string & body, + const Strings & body_secrets = {} ) - : Exception(makeExceptionMessage(code, uri, http_status_, reason, body)) + : Exception(makeExceptionMessage(code, uri, http_status_, reason, body, body_secrets)) , http_status(http_status_) {} @@ -47,7 +49,8 @@ class HTTPException : public Exception const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus http_status, const std::string & reason, - const std::string & body); + const std::string & body, + const Strings & body_secrets); const char * name() const noexcept override { return "DB::HTTPException"; } const char * className() const noexcept override { return "DB::HTTPException"; } @@ -76,7 +79,15 @@ bool isRedirect(Poco::Net::HTTPResponse::HTTPStatus status); std::istream * receiveResponse( Poco::Net::HTTPClientSession & session, const Poco::Net::HTTPRequest & request, Poco::Net::HTTPResponse & response, bool allow_redirects); +/// Returns the Bearer token carried by a request's `Authorization` header, so a caller can scrub it +/// from an error response body that reflects it back. A `Basic` header is not decoded, so a password +/// is not scrubbed. Empty when there is no bearer token. +Strings requestCredentialSecrets(const Poco::Net::HTTPRequest & request); + +/// `body_secrets` are credential strings to scrub from the reflected response body (see +/// `requestCredentialSecrets`): a remote error body can echo the request's own credentials. void assertResponseIsOk( - const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false); + const String & uri, Poco::Net::HTTPResponse & response, std::istream & istr, bool allow_redirects = false, + const Strings & body_secrets = {}); } diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index 2ff90b64dcd3d..02834987d5387 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,15 @@ Poco::URI getUriAfterRedirect(const Poco::URI & prev_uri, Poco::Net::HTTPRespons return location_uri; } +/// A URI that ends up in a log line or an exception message must never carry credentials. Convenience +/// overload for a `Poco::URI`: masks the userinfo and presigned-URL parameters via the shared helper. +std::string maskedURI(const Poco::URI & uri) +{ + std::string masked = uri.toString(); + DB::maskURICredentials(masked); + return masked; +} + class ReadBufferFromSessionResponse : public DB::ReadBufferFromIStream { private: @@ -171,7 +181,7 @@ bool ReadWriteBufferFromHTTP::checkIfActuallySeekable() String ReadWriteBufferFromHTTP::getFileName() const { - return initial_uri.toString(); + return maskedURI(initial_uri); } void ReadWriteBufferFromHTTP::getHeadResponse(Poco::Net::HTTPResponse & response) @@ -280,7 +290,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callImpl( auto & resp_stream = session->receiveResponse(response); - assertResponseIsOk(current_uri.toString(), response, resp_stream, allow_redirects); + assertResponseIsOk(maskedURI(current_uri), response, resp_stream, allow_redirects, requestCredentialSecrets(request)); return ReadWriteBufferFromHTTP::CallResult(std::move(session), resp_stream); } @@ -301,7 +311,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callWithRedirects( " You can {} redirects by changing the setting 'max_http_get_redirects'." " Example: `SET max_http_get_redirects = 10`." " Redirects are restricted to prevent possible attack when a malicious server redirects to an internal resource, bypassing the authentication or firewall.", - initial_uri.toString(), max_redirects ? "increase the allowed maximum number of" : "allow"); + maskedURI(initial_uri), max_redirects ? "increase the allowed maximum number of" : "allow"); current_uri = uri_redirect; result = callImpl(response, method_, range, true); @@ -374,7 +384,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, "Failed to make request to '{}'{}. " "Error: '{}'. " "Failed at try {}/{}.", - initial_uri.toString(), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), + maskedURI(initial_uri), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", maskedURI(current_uri)), error_message, attempt, read_settings.http_max_tries); @@ -391,7 +401,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, "Error: {}. " "Failed at try {}/{}. " "Will retry with current backoff wait is {}/{} ms.", - initial_uri.toString(), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), + maskedURI(initial_uri), current_uri.toString() == initial_uri.toString() ? String() : fmt::format(" redirect to '{}'", maskedURI(current_uri)), error_message, attempt + 1, read_settings.http_max_tries, milliseconds_to_wait, read_settings.http_retry_max_backoff_ms); @@ -429,7 +439,7 @@ std::unique_ptr ReadWriteBufferFromHTTP::initialize() /// it is retriable error throw HTTPException( ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, - current_uri.toString(), + maskedURI(current_uri), Poco::Net::HTTPResponse::HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, response.getReason(), explanation); @@ -549,13 +559,13 @@ size_t ReadWriteBufferFromHTTP::readBigAt(char * to, size_t n, size_t offset, co String explanation = fmt::format( "When reading with readBigAt {}." "Cannot read with range: [{}, {}] (response status: {}, reason: {}), will retry", - initial_uri.toString(), + maskedURI(initial_uri), *range.begin, *range.end, toString(response.getStatus()), response.getReason()); throw HTTPException( ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, - current_uri.toString(), + maskedURI(current_uri), Poco::Net::HTTPResponse::HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, response.getReason(), explanation); diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index dc95d582fd8a0..4edb13d7bafea 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -46,7 +47,10 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( request.set("Content-Type", content_type); } - LOG_TRACE((getLogger("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); + /// Mask credentials before logging: a URL sink would otherwise log them on every INSERT. + std::string uri_for_logging = uri.toString(); + maskURICredentials(uri_for_logging); + LOG_TRACE((getLogger("WriteBufferToHTTP")), "Sending request to {}", uri_for_logging); ProfileEvents::increment(ProfileEvents::WriteBufferFromHTTPRequestsSent); ostr = &session->sendRequest(request); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 9f2fb9d55c3f1..0e4881567fa72 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include @@ -942,7 +943,14 @@ namespace format = format_name; } - String getLastFilePath() const override { return current_url_option; } + String getLastFilePath() const override + { + /// `current_url_option` is kept raw for requests and schema cache keys, but the diagnostic + /// path can reach the client, server log and `system.query_log`, so mask credentials here. + String masked = current_url_option; + maskURICredentials(masked); + return masked; + } bool supportsLastReadBufferRecreation() const override { return true; } diff --git a/src/Storages/System/StorageSystemSchemaInferenceCache.cpp b/src/Storages/System/StorageSystemSchemaInferenceCache.cpp index 348a38c613d7b..8a8a045db4315 100644 --- a/src/Storages/System/StorageSystemSchemaInferenceCache.cpp +++ b/src/Storages/System/StorageSystemSchemaInferenceCache.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,27 @@ namespace DB { +/// Whether a value starts with an RFC 3986 scheme followed by `://` (`^[a-zA-Z][a-zA-Z0-9+.-]*://`). +/// `Common/maskURIPassword.h` has no `findURIAuthority` on this release; this is the same check. +static bool startsWithURIScheme(std::string_view uri) +{ + static constexpr std::string_view SEPARATOR = "://"; + + /// The character classes are spelled out rather than taken from ``, which depends on the locale. + auto is_letter = [](char c) { return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); }; + auto is_letter_or_digit = [&](char c) { return is_letter(c) || ('0' <= c && c <= '9'); }; + + if (uri.empty() || !is_letter(uri[0])) + return false; + + size_t scheme_end = 1; + while (scheme_end < uri.length() + && (is_letter_or_digit(uri[scheme_end]) || uri[scheme_end] == '+' || uri[scheme_end] == '.' || uri[scheme_end] == '-')) + ++scheme_end; + + return uri.compare(scheme_end, SEPARATOR.length(), SEPARATOR) == 0; +} + static String getSchemaString(const ColumnsDescription & columns) { WriteBufferFromOwnString buf; @@ -58,7 +80,12 @@ static void fillDataImpl(MutableColumns & res_columns, SchemaCache & schema_cach for (const auto & [key, schema_info] : s3_schema_cache_data) { res_columns[0]->insert(storage_name); - res_columns[1]->insert(key.source); + String source = key.source; + /// Only a URL can carry a credential. A `File` key is a local path, where the presigned + /// parameter scan would rewrite a '?' in a file name and swallow everything after it. + if (startsWithURIScheme(source)) + maskURICredentials(source); + res_columns[1]->insert(source); res_columns[2]->insert(key.format); res_columns[3]->insert(key.additional_format_info); res_columns[4]->insert(schema_info.registration_time); diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference new file mode 100644 index 0000000000000..9f0c48987f44c --- /dev/null +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference @@ -0,0 +1,9 @@ +url_function: OK masked +url_status_failure: OK masked +insert_presigned: OK masked +dictionary_reload: OK masked +show_create: OK masked +schema_inference_cache: OK masked +schema_inference_error: OK masked +basic_header_passthrough: OK +body_not_overmasked: OK diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh new file mode 100755 index 0000000000000..b0906c63fdef3 --- /dev/null +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A credential embedded in an HTTP URI must never appear in an error message, in SHOW CREATE output, +# or in system.query_log. Only the masked form scheme://[HIDDEN]@host (or X-Amz-Signature=[HIDDEN] for +# a presigned URL) may be shown. + +PW="pwleakprobe9f2a" +URI="http://leakuser:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping" +USERINFO_SHAPE="[HIDDEN]@${CLICKHOUSE_HOST}" + +# Reads text from stdin and asserts the credential was present but masked: the cleartext secret is +# absent and the required masked shape is present. Asserting the shape (not just that "[HIDDEN]" appears +# somewhere) means a text that never carried the credential fails, so the check cannot pass vacuously. +assert_shape() { + local label="$1" secret="$2" shape="$3" text + text=$(cat) + if echo "$text" | grep -qF "$secret"; then + echo "$label: FAIL cleartext" + elif echo "$text" | grep -qF "$shape"; then + echo "$label: OK masked" + else + echo "$label: FAIL uri absent" + fi +} + +# 1. url() table function: the /ping response ("Ok.") fails to parse as CSV, so the URI is appended to +# the exception as "(in file/uri ...)". Grep that line out of the exception - the client also echoes +# the user's own submitted query, which legitimately contains what the user typed. +${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('${URI}', 'CSV', 'id UInt64, val String')" 2>&1 \ + | grep -F 'in file/uri' | assert_shape "url_function" "$PW" "$USERINFO_SHAPE" + +# 1b. An HTTP status failure (non-2xx) is reported by assertResponseIsOk as "Received error from +# remote server ", a different code path than the CSV-parse suffix above. A request to an +# unknown path returns 404, so the URI in that exception must also be masked. +URI_404="http://leakuser:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/no_such_handler_${CLICKHOUSE_TEST_UNIQUE_NAME}" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('${URI_404}', 'CSV', 'id UInt64, val String')" 2>&1 \ + | grep -F 'Received error from remote server' | assert_shape "url_status_failure" "$PW" "$USERINFO_SHAPE" + +# 1c. INSERT INTO url() writes through WriteBufferFromHTTP - the only path here that does - and a +# presigned URL carries its credential in the query parameters, not the userinfo. A request with a +# missing ?database returns 404 whose body does not echo the URI, so the signature can only appear +# through the URI in the exception, which must be masked to X-Amz-Signature=[HIDDEN]. +SIG="sigprobe${CLICKHOUSE_TEST_UNIQUE_NAME}" +URI_SIG="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?database=no_such_db_${CLICKHOUSE_TEST_UNIQUE_NAME}&X-Amz-Signature=${SIG}" +${CLICKHOUSE_CLIENT} --query "INSERT INTO TABLE FUNCTION url('${URI_SIG}', 'CSV', 'id UInt64') VALUES (1)" 2>&1 \ + | grep -F 'Received error from remote server' | assert_shape "insert_presigned" "$SIG" "X-Amz-Signature=[HIDDEN]" + +# 2. A dictionary whose HTTP source URL carries credentials. +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS dict_uri_leak" +${CLICKHOUSE_CLIENT} --query "CREATE DICTIONARY dict_uri_leak (id UInt64, val String) PRIMARY KEY id SOURCE(HTTP(url '${URI}' format 'CSV')) LAYOUT(FLAT(SIZE_IN_CELLS 100)) LIFETIME(0)" + +# 2a. Reloading fails to parse; the URI must be masked in the exception. +${CLICKHOUSE_CLIENT} --query "SYSTEM RELOAD DICTIONARY dict_uri_leak" 2>&1 \ + | grep -F 'in file/uri' | assert_shape "dictionary_reload" "$PW" "$USERINFO_SHAPE" + +# 2b. SHOW CREATE must mask the password. +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DICTIONARY dict_uri_leak" 2>&1 | assert_shape "show_create" "$PW" "$USERINFO_SHAPE" + +# 3. system.query_log must store neither the query text nor the exception with the cleartext password. +# Print the offending rows rather than a count: on success this selects nothing (the reference has +# no line here), and any row printed on failure is the culprit - its query and exception show what +# leaked. The needle is split so that this checking query does not itself contain the contiguous +# secret. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT type, query, exception + FROM system.query_log + WHERE event_date >= yesterday() + AND current_database = currentDatabase() + AND (query LIKE '%' || 'pwleakprobe' || '9f2a%' OR exception LIKE '%' || 'pwleakprobe' || '9f2a%') + FORMAT Vertical" + +# 4. Schema inference stores the source URL in system.schema_inference_cache; the credential in its +# userinfo must be masked there too. Inferring a schema from a credentialed /ping URL (which needs +# no auth and whose "Ok." body is inferrable as one column) populates the cache. +SIC_SECRET="sicpwprobe${CLICKHOUSE_TEST_UNIQUE_NAME}" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://leakuser:${SIC_SECRET}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping', 'CSV')" >/dev/null 2>&1 +${CLICKHOUSE_CLIENT} --query "SELECT source FROM system.schema_inference_cache WHERE storage = 'URL'" \ + | assert_shape "schema_inference_cache" "$SIC_SECRET" "[HIDDEN]@${CLICKHOUSE_HOST}" + +# 4b. When schema inference fails, the source URL is appended to the error as "(in file/uri ...)", and it +# must be masked there too. The "Ok." body of /ping is not JSON, so inferring JSONEachRow fails. +SIE_SECRET="siepwprobe${CLICKHOUSE_TEST_UNIQUE_NAME}" +${CLICKHOUSE_CLIENT} --query "DESC url('http://leakuser:${SIE_SECRET}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping', 'JSONEachRow')" 2>&1 \ + | grep -F 'in file/uri' | assert_shape "schema_inference_error" "$SIE_SECRET" "(in file/uri http://[HIDDEN]@${CLICKHOUSE_HOST}" + +# 5. A header value is never decoded as a credential. An 'Authorization: Basic ' whose payload +# is not valid base64 (here supplied through headers()) must reach the endpoint verbatim, not fail +# while the request is inspected for secrets to scrub. /ping needs no auth and answers "Ok.". +A_OUT=$(${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping', 'LineAsString', 'line String', headers('Authorization' = 'Basic ghp_AbCd1234'))" 2>&1) +if echo "$A_OUT" | grep -qF 'Ok.'; then echo "basic_header_passthrough: OK"; else echo "basic_header_passthrough: FAIL"; fi + +# 6. A URL userinfo is sent as 'Authorization: Basic base64(user:password)'. Recovering the user name +# from that header to scrub the error body would replace a one-letter user ('a') everywhere it +# occurs, turning the remote "Authentication failed" body into "Authentic[HIDDEN]tion f[HIDDEN]iled". +# The body must be preserved, and the password must not appear. Authenticating against ClickHouse's +# own HTTP port with a bad password returns that body. Grep the "Received error from remote server" +# line out of the exception: the client also echoes the user's own submitted query, which +# legitimately contains the password the user typed. +B_OUT=$(${CLICKHOUSE_CLIENT} --query "SELECT * FROM url('http://a:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+1', 'CSV', 'x UInt8')" 2>&1 \ + | grep -F 'Received error from remote server') +if echo "$B_OUT" | grep -qF "$PW"; then + echo "body_not_overmasked: FAIL cleartext" +elif echo "$B_OUT" | grep -qF 'Authentication failed'; then + echo "body_not_overmasked: OK" +else + echo "body_not_overmasked: FAIL mangled" +fi + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS dict_uri_leak" From 2844f040b60033e58bce6513b018d0f77686818c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 13:45:04 +0000 Subject: [PATCH 15/46] Update autogenerated version to 26.3.56.2 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 8677193ae1330..22f90d883026e 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54563) +SET(VERSION_REVISION 54564) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 56) -SET(VERSION_GITHASH e029f7835cc3f7d6e22db137c01d75c30f855893) -SET(VERSION_DESCRIBE v26.3.56.1-lts) -SET(VERSION_STRING 26.3.56.1) +SET(VERSION_PATCH 57) +SET(VERSION_GITHASH ff8f6efef429ef44bc1ffb20ef189d164612600f) +SET(VERSION_DESCRIBE v26.3.57.1-lts) +SET(VERSION_STRING 26.3.57.1) # end of autochange From 1598d9a11b7d4b1c9d60d0bdddf31e6ba3b95a56 Mon Sep 17 00:00:00 2001 From: yariks5s Date: Mon, 19 Oct 2026 13:52:51 +0000 Subject: [PATCH 16/46] Adapt the tests to 26.3 --- ..._external_engine_requires_select.reference | 6 +-- ...eate_as_external_engine_requires_select.sh | 2 - ..._settings_secret_requires_select.reference | 6 --- ..._engine_settings_secret_requires_select.sh | 43 ------------------- ...as_time_series_inner_engine_credentials.sh | 2 +- 5 files changed, 3 insertions(+), 56 deletions(-) delete mode 100644 tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference delete mode 100755 tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh diff --git a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference index f0edc2dad6cfa..30e2b3bcc8622 100644 --- a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference +++ b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.reference @@ -8,14 +8,12 @@ necessary to have the grant SELECT ON db.function_src -- copy_of_url_src_no_password: URL -- copy_of_function_src_no_password: -Proxy --- null_copy_of_url_src: -Null +StorageProxy after GRANT SELECT: -- copy_of_url_src: URL -- copy_of_function_src: -Proxy +StorageProxy without SHOW COLUMNS: -- local_src: necessary to have the grant SHOW COLUMNS ON db.local_src diff --git a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh index 22f7199b241af..e6ebd150d0b23 100755 --- a/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh +++ b/tests/queries/0_stateless/05141_create_as_external_engine_requires_select.sh @@ -50,8 +50,6 @@ try_copy copy_of_function_src function_src # these two hold nothing masked, so the server copies them as before try_copy copy_of_url_src_no_password url_src_no_password try_copy copy_of_function_src_no_password function_src_no_password -# Null replaces the engine here, so the copy inherits nothing masked -try_copy null_copy_of_url_src url_src --restore_replace_external_engines_to_null 1 echo "after GRANT SELECT:" ${CLICKHOUSE_CLIENT} -q " diff --git a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference deleted file mode 100644 index 5a636ec83f4c1..0000000000000 --- a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.reference +++ /dev/null @@ -1,6 +0,0 @@ -with SHOW COLUMNS only: -necessary to have the grant SELECT ON db.kafka_src -with a password in this query: -Kafka -after GRANT SELECT: -Kafka diff --git a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh b/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh deleted file mode 100755 index e810ad7954ede..0000000000000 --- a/tests/queries/0_stateless/05142_create_as_engine_settings_secret_requires_select.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest -# no-fasttest: the fast test does not build the Kafka engine - -# the engine settings can hold credentials, not only the engine arguments. SHOW CREATE TABLE masks -# kafka_sasl_password too, so a copy of it also needs SELECT. - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -user="user_${CLICKHOUSE_DATABASE}" -db="${CLICKHOUSE_DATABASE}" - -# the server does not contact the broker. it only copies the definition -${CLICKHOUSE_CLIENT} -q " - DROP USER IF EXISTS ${user}; - CREATE USER ${user}; - GRANT CREATE TABLE ON ${db}.* TO ${user}; - GRANT KAFKA ON *.* TO ${user}; - - CREATE TABLE ${db}.kafka_src (id UInt64) ENGINE = Kafka - SETTINGS kafka_broker_list = 'localhost:9092', kafka_topic_list = 'topic', - kafka_group_name = 'group', kafka_format = 'CSV', kafka_sasl_password = 'secret'; - - GRANT SHOW COLUMNS ON ${db}.kafka_src TO ${user}; -" - -echo "with SHOW COLUMNS only:" -${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.copy_of_kafka_src AS ${db}.kafka_src" 2>&1 \ - | grep -oE "necessary to have the grant [A-Z ]+ ON ${db}\.[a-z_]+" | head -n 1 | sed "s/${db}/db/" - -# the copy does not inherit a setting that this query gives, so it needs nothing more -echo "with a password in this query:" -${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.own_password AS ${db}.kafka_src SETTINGS kafka_sasl_password = 'own'" -${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = 'own_password'" - -echo "after GRANT SELECT:" -${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON ${db}.kafka_src TO ${user}" -${CLICKHOUSE_CLIENT} --user "${user}" -q "CREATE TABLE ${db}.copy_of_kafka_src AS ${db}.kafka_src" -${CLICKHOUSE_CLIENT} -q "SELECT engine FROM system.tables WHERE database = '${db}' AND name = 'copy_of_kafka_src'" - -${CLICKHOUSE_CLIENT} -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh index e110640472135..867745e8ea1ad 100755 --- a/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh +++ b/tests/queries/0_stateless/05331_create_as_time_series_inner_engine_credentials.sh @@ -21,7 +21,7 @@ client=(${CLICKHOUSE_CLIENT} --allow_experimental_time_series_table 1) TABLE ENGINE ON ReplacingMergeTree, TABLE ENGINE ON URL TO ${user}; CREATE TABLE ${db}.ts_src ENGINE = TimeSeries - SAMPLES ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); + DATA ENGINE = URL('http://user:password@127.0.0.1:1/', 'CSV'); GRANT SHOW COLUMNS ON ${db}.ts_src TO ${user}; " From 3a51cb4e8ba836ee0186da75e7fd3efba551a262 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 14:07:59 +0000 Subject: [PATCH 17/46] Backport #125785 to 26.3: Use constant-time comparison for TOTP codes and plaintext passwords --- src/Access/Authentication.cpp | 3 ++- src/Access/Common/OneTimePassword.cpp | 13 +++++++++---- src/Common/StringUtils.h | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index a0bcd3479becc..fa33d8eca45bf 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -17,6 +17,7 @@ #include #include +#include #if USE_SSL # include @@ -205,7 +206,7 @@ namespace case AuthenticationType::PLAINTEXT_PASSWORD: { const auto & password_plaintext = authentication_method.getPasswordHashBinary(); - return Util::stringToDigest(password) == password_plaintext ? on_success : Authentication::CredentialsCheckResult::Fail; + return constantTimeEquals(Util::stringToDigest(password), password_plaintext) ? on_success : Authentication::CredentialsCheckResult::Fail; } case AuthenticationType::SHA256_PASSWORD: { diff --git a/src/Access/Common/OneTimePassword.cpp b/src/Access/Common/OneTimePassword.cpp index 50ac6477efd26..6861170dbf22e 100644 --- a/src/Access/Common/OneTimePassword.cpp +++ b/src/Access/Common/OneTimePassword.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -151,15 +152,19 @@ String getOneTimePassword(const String & secret [[ maybe_unused ]], const OneTim static std::optional findMatchingTimeStep(std::string_view password, const OneTimePasswordSecret & secret, UInt64 current_time) { if (password.size() != static_cast(secret.params.num_digits) - || !std::all_of(password.begin(), password.end(), ::isdigit)) + || !std::all_of(password.begin(), password.end(), isNumericASCII)) return {}; + std::optional result; for (int delta : {0, -1, 1}) { - if (password == getOneTimePassword(secret.key, secret.params, current_time + delta * secret.params.period)) - return static_cast(current_time / secret.params.period) + delta; + auto candidate_pass = getOneTimePassword(secret.key, secret.params, current_time + delta * secret.params.period); + bool matched = constantTimeEquals(password, candidate_pass); + /// No early return to avoid leaking which window matched + if (matched && !result.has_value()) + result = static_cast(current_time / secret.params.period) + delta; } - return {}; + return result; } bool checkOneTimePassword(std::string_view password, const OneTimePasswordSecret & secret) diff --git a/src/Common/StringUtils.h b/src/Common/StringUtils.h index d1b236c92a20a..cfe8f76fba1eb 100644 --- a/src/Common/StringUtils.h +++ b/src/Common/StringUtils.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -272,6 +273,30 @@ inline bool equalsCaseInsensitive(std::string_view a, std::string_view b) && std::equal(a.begin(), a.end(), b.begin(), [](char x, char y) { return equalsCaseInsensitive(x, y); }); } +/// Equality for secret-derived byte sequences such as passwords and one-time codes. Unlike `memcmp` +/// and `operator==`, the runtime does not depend on the position of the first mismatch, so timing does +/// not reveal how many leading bytes were right. The lengths are treated as public, as in OpenSSL's +/// `CRYPTO_memcmp`. The `volatile` reads keep the compiler from turning the loop into an early-exit compare. +inline bool constantTimeEquals(std::span lhs, std::span rhs) +{ + if (lhs.size() != rhs.size()) + return false; + + const volatile uint8_t * a = lhs.data(); + const volatile uint8_t * b = rhs.data(); + uint8_t diff = 0; + for (size_t i = 0; i < lhs.size(); ++i) + diff |= a[i] ^ b[i]; + return diff == 0; +} + +inline bool constantTimeEquals(std::string_view lhs, std::string_view rhs) +{ + return constantTimeEquals( + std::span(reinterpret_cast(lhs.data()), lhs.size()), + std::span(reinterpret_cast(rhs.data()), rhs.size())); +} + template std::string trim(const std::string & str, F && predicate) From a300aeff3f2c82e488d350b35c7475762702cfcc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 14:22:09 +0000 Subject: [PATCH 18/46] Backport #126120 to 26.3: Fix out-of-bounds accesses in the replxx line editor --- contrib/replxx | 2 +- .../05401_client_history_search_crash.expect | 91 +++++++++++++++++++ ...5401_client_history_search_crash.reference | 0 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100755 tests/queries/0_stateless/05401_client_history_search_crash.expect create mode 100644 tests/queries/0_stateless/05401_client_history_search_crash.reference diff --git a/contrib/replxx b/contrib/replxx index 7ecd397fd2aea..bcab1537d03d7 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 7ecd397fd2aeabcad29c05fbf7ae8a2a83c3dede +Subproject commit bcab1537d03d7645ebc9ce0327ec6095416b79c3 diff --git a/tests/queries/0_stateless/05401_client_history_search_crash.expect b/tests/queries/0_stateless/05401_client_history_search_crash.expect new file mode 100755 index 0000000000000..48852f89ae896 --- /dev/null +++ b/tests/queries/0_stateless/05401_client_history_search_crash.expect @@ -0,0 +1,91 @@ +#!/usr/bin/expect -f + +set basedir [file dirname $argv0] +set basename [file tail $argv0] +if {[info exists env(CLICKHOUSE_TMP)]} { + set CLICKHOUSE_TMP $env(CLICKHOUSE_TMP) +} else { + set CLICKHOUSE_TMP "." +} +exp_internal -f $CLICKHOUSE_TMP/$basename.debuglog 0 +set history_file $CLICKHOUSE_TMP/$basename.history + +log_user 0 +set timeout 60 +match_max 100000 + +expect_after { + # Do not ignore eof from expect + -i $any_spawn_id eof { exp_continue } + # A default timeout action is to do nothing, change it to fail + -i $any_spawn_id timeout { exit 1 } +} + +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file" +expect ":) " + +send -- "SELECT 1\r" +expect "1" +expect ":) " + +# Recall the query (Ctrl-P) and make it longer than the history entry. +send -- "\x10" +expect "SELECT 1" +send -- "23" +expect "SELECT 123" + +# A forward search (Ctrl-S) that finds nothing, left with Ctrl-F, must keep the edited line and the cursor in it. +send -- "\x13" +expect "(i-search)" +send -- "\"" +expect "(i-search)`\"'" +send -- "\x06" +expect ":) " +send -- "+4\r" +expect -re {\s127\s} +expect ":) " + +# Paste a query with a tab, then start a search seeded with the line (Meta-R), which copies the tab into the search prompt. +send -- "\033\[200~SELECT\t40 + 2\033\[201~" +expect "SELECT 40 + 2" +send -- "\033r" +expect "(reverse-i-search)" +send -- "\x07" +expect ":) " +send -- "\r" +expect -re {\s42\s} +expect ":) " + +send -- "\4" +expect eof + +# Leave a history search with a prompt that contains a double-width character. +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file --prompt '日 :) '" +expect "日 :) " +send -- "\x14" +expect "(reverse-i-search)" +send -- "x" +send -- "\x07" +expect "日 :) " +send -- "SELECT 50 - 8\r" +expect -re {\s42\s} +expect "日 :) " +send -- "\4" +expect eof + +# Ctrl-W and Alt-. must keep U+10FFFF inside a word, sent as raw UTF-8 bytes (Tcl 8.6 needs a binary channel for that). +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_EXPECT_OPT --history_file=$history_file" +fconfigure $spawn_id -encoding binary +expect ":) " +send -- "SELECT 'a\xf4\x8f\xbf\xbfz" +send -- "\x17" +send -- "41 + 1 AS `b\xf4\x8f\xbf\xbf`\r" +expect -re {\s42\s} +expect ":) " +send -- "SELECT 6 * 7 AS " +send -- "\033." +send -- "\r" +expect -re {\s42\s} +expect ":) " +send -- "\4" +expect eof diff --git a/tests/queries/0_stateless/05401_client_history_search_crash.reference b/tests/queries/0_stateless/05401_client_history_search_crash.reference new file mode 100644 index 0000000000000..e69de29bb2d1d From e640778cb77671c76ddffffcbdbe04cd87a63fd7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 14:34:21 +0000 Subject: [PATCH 19/46] Backport #113877 to 26.3: Fix out-of-bounds write on a long codec chain --- src/Compression/CompressionCodecGorilla.cpp | 5 +- src/Compression/CompressionCodecMultiple.cpp | 62 +++++- .../CompressionFactoryAdditions.cpp | 58 ++++++ src/Storages/AlterCommands.cpp | 4 +- ...812_codec_chain_reserve_overflow.reference | 14 ++ .../04812_codec_chain_reserve_overflow.sql | 176 ++++++++++++++++++ 6 files changed, 311 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference create mode 100644 tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index e43ef489ab1e5..73896def3b0a3 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -166,9 +166,10 @@ UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) { const UInt32 items_count = uncompressed_size / data_bytes_size; - static const auto DATA_BIT_LENGTH = getBitLengthOfLength(data_bytes_size); + /// Not static: data_bytes_size is an argument, so a static keeps the first caller's width. + const auto DATA_BIT_LENGTH = getBitLengthOfLength(data_bytes_size); // -1 since there must be at least 1 non-zero bit. - static const auto LEADING_ZEROES_BIT_LENGTH = DATA_BIT_LENGTH - 1; + const auto LEADING_ZEROES_BIT_LENGTH = DATA_BIT_LENGTH - 1; // worst case (for 32-bit value): // 11 + 5 bits of leading zeroes bit-size + 5 bits of data bit-size + non-zero data bits. diff --git a/src/Compression/CompressionCodecMultiple.cpp b/src/Compression/CompressionCodecMultiple.cpp index 8bb2cfdd6a173..2c47d17fdefa1 100644 --- a/src/Compression/CompressionCodecMultiple.cpp +++ b/src/Compression/CompressionCodecMultiple.cpp @@ -4,11 +4,37 @@ #include #include #include +#include + +#include namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_COMPRESS; +} + +namespace +{ + +/// A codec never reserves less than its own input, so a result below the input means the UInt32 +/// arithmetic wrapped. The check does not depend on where inside the callee the wrap happened. +UInt32 getCheckedReserveSize(const CompressionCodecPtr & codec, UInt32 size, size_t codec_index, size_t codecs_count) +{ + UInt32 reserve_size = codec->getCompressedReserveSize(size); + if (reserve_size < size) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Too many codecs in the codec chain: the size reserved for compressing {} bytes overflows 4 GiB " + "at codec {} of {} ({}). Use fewer codecs.", + size, codec_index + 1, codecs_count, codec->getCodecDesc()->formatForErrorMessage()); + return reserve_size; +} + +} + CompressionCodecMultiple::CompressionCodecMultiple(Codecs codecs_) : codecs(codecs_) { @@ -32,16 +58,36 @@ void CompressionCodecMultiple::updateHash(SipHash & hash) const UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_size) const { + /// doCompressData stores the number of codecs in one byte, and doDecompressData reads it back + /// from there, so a longer chain would write a part that cannot be read. + if (codecs.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Too many codecs in the codec chain: {}. The number of codecs is stored in one byte, " + "so at most {} are supported.", + codecs.size(), static_cast(std::numeric_limits::max())); + UInt32 compressed_size = uncompressed_size; - for (const auto & codec : codecs) - compressed_size = codec->getCompressedReserveSize(compressed_size); + for (size_t idx = 0; idx < codecs.size(); ++idx) + compressed_size = getCheckedReserveSize(codecs[idx], compressed_size, idx, codecs.size()); /// TotalCodecs ByteForEachCodec data - return static_cast(sizeof(UInt8) + codecs.size() + compressed_size); + size_t total_size = sizeof(UInt8) + codecs.size() + compressed_size; + /// getCompressedReserveSize adds getHeaderSize() to this in UInt32, so leave room for it. + static constexpr size_t max_total_size = std::numeric_limits::max() - ICompressionCodec::getHeaderSize(); + if (total_size > max_total_size) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Too many codecs in the codec chain: the size reserved for compressing {} bytes ({}) exceeds 4 GiB. " + "Use fewer codecs.", + uncompressed_size, total_size); + + return static_cast(total_size); } UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 source_size, char * dest) const { + /// The caller sized dest from getMaxCompressedDataSize(source_size). + const UInt32 dest_size = getMaxCompressedDataSize(source_size); + PODArray compressed_buf; PODArray uncompressed_buf(source, source + source_size); @@ -52,7 +98,7 @@ UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 sour { const auto codec = codecs[idx]; dest[codecs_byte_pos] = codec->getMethodByte(); - compressed_buf.resize(codec->getCompressedReserveSize(source_size)); + compressed_buf.resize(getCheckedReserveSize(codec, source_size, idx, codecs.size())); UInt32 size_compressed = codec->compress(uncompressed_buf.data(), source_size, compressed_buf.data()); @@ -60,9 +106,15 @@ UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 sour source_size = size_compressed; } + /// source_size is now each codec's actual output, computed independently of the bounds above. + size_t written_size = sizeof(UInt8) + codecs.size() + source_size; + if (written_size > dest_size) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Compressed data of size {} does not fit the reserved buffer of size {}", written_size, dest_size); + memcpy(&dest[1 + codecs.size()], uncompressed_buf.data(), source_size); - return static_cast(1 + codecs.size() + source_size); + return static_cast(written_size); } UInt32 CompressionCodecMultiple::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index 3c4f6230ff25b..6813db2eeec63 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -19,6 +19,9 @@ #include #include #include +#include + +#include namespace DB @@ -83,6 +86,22 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( { ASTPtr codecs_descriptions = make_intrusive(); + /// A codec that depends on the data type resolves differently per substream, and every + /// substream is compressed with its own chain, so there is one chain per substream. + size_t num_substreams = 0; + if (column_type) + { + ISerialization::StreamCallback count_callback = [&](const auto & substream_path) + { + if (ISerialization::isSpecialCompressionAllowed(substream_path)) + ++num_substreams; + }; + column_type->getDefaultSerialization()->enumerateStreams(count_callback, column_type); + } + /// A codec resolved without a data type is one chain, and so is a type whose substreams all + /// refuse a special codec. + VectorWithMemoryTracking codec_chains(num_substreams ? num_substreams : 1); + bool with_compression_codec = false; bool with_none_codec = false; std::optional first_generic_compression_codec_pos; @@ -126,6 +145,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( if (column_type) { CompressionCodecPtr prev_codec; + size_t substream_index = 0; ISerialization::StreamCallback callback = [&](const auto & substream_path) { assert(!substream_path.empty()); @@ -134,6 +154,12 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( const auto & last_type = substream_path.back().data.type; result_codec = getImpl(codec_family_name, codec_arguments, last_type.get()); + /// Enumeration order is the same for every codec of the chain, so the + /// index identifies the substream. + if (substream_index < codec_chains.size()) + codec_chains[substream_index].push_back(result_codec); + ++substream_index; + /// Case for column Tuple, which compressed with codec which depends on data type, like Delta. /// We cannot substitute parameters for such codecs. if (prev_codec && prev_codec->getHash() != result_codec->getHash()) @@ -162,6 +188,11 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); } + /// A codec that was not resolved per substream is the same one for all of them. + for (auto & chain : codec_chains) + if (chain.size() == i) + chain.push_back(result_codec); + with_compression_codec |= result_codec->isCompression(); with_none_codec |= result_codec->isNone(); @@ -182,6 +213,33 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( if (sanity_check) { + /// CompressionCodecMultiple stores the number of codecs in a single byte, so a longer chain + /// describes a part that cannot be read back. Being a sanity check, this is not enforced on + /// the metadata-load path, where an already stored table must stay loadable. + if (codecs_descriptions->children.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Too many codecs in the codec chain: {}. The number of codecs is stored in one byte, " + "so at most {} are supported.", + codecs_descriptions->children.size(), static_cast(std::numeric_limits::max())); + + /// A codec never reserves less than its input, so a reserve below the input means the + /// UInt32 compounding wrapped. One byte is the weakest block there is: a chain that + /// wraps on it cannot compress a block of any size. + for (const auto & chain : codec_chains) + { + UInt32 reserve_size = 1; + for (size_t i = 0; i < chain.size(); ++i) + { + const UInt32 next_reserve_size = chain[i]->getCompressedReserveSize(reserve_size); + if (next_reserve_size < reserve_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Too many codecs in the codec chain: {}. The size they reserve for compressing a block " + "overflows 4 GiB at codec {} ({}), so no block could be compressed. Use fewer codecs.", + chain.size(), i + 1, chain[i]->getCodecDesc()->formatForErrorMessage()); + reserve_size = next_reserve_size; + } + } + if (codecs_descriptions->children.size() > 1 && with_none_codec) throw Exception(ErrorCodes::BAD_ARGUMENTS, "It does not make sense to have codec NONE along with other compression codecs: {}. " diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 11f569c1a7eb9..b1661cffc3764 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1554,9 +1554,11 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { if (all_columns.hasAlias(column_name)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); + /// The type is optional here, and a codec can resolve differently per type, so + /// validate against the type the column will have, as `apply` does. CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( command.codec, - command.data_type, + command.data_type ? command.data_type : all_columns.get(column_name).type, !context->getSettingsRef()[Setting::allow_suspicious_codecs], context->getSettingsRef()[Setting::allow_experimental_codecs]); } diff --git a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference new file mode 100644 index 0000000000000..15a39899594a5 --- /dev/null +++ b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.reference @@ -0,0 +1,14 @@ +5000 0 +0 +4 +5000 0 +0 +2500 0 +0 +5000 0 +0 +5000 0 +0 +0 +0 +5000 0 diff --git a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql new file mode 100644 index 0000000000000..f047a1d4dc1af --- /dev/null +++ b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql @@ -0,0 +1,176 @@ +-- Tags: no-fasttest, no-parallel +-- Tag no-fasttest: a codec chain long enough to overflow a 32-bit reserve needs a second or two. +-- Tag no-parallel: the 104-stage arm below allocates just under 4 GiB against a per-user memory +-- limit that concurrent copies of this test share, and that SET max_memory_usage = 0 does not lift. +-- Random settings limits: min_columns_to_activate_adaptive_write_buffer=(3, None) + +-- CompressionCodecMultiple compounded each stage's reserved size in a UInt32 without checking for +-- overflow. Every codec reserves more than its input, so a long enough chain wrapped the reserve +-- below its own input, and the destination buffer was then sized from the wrapped number while the +-- real payload kept growing: an out-of-bounds heap write, and a server crash. Such a chain is +-- rejected when the table is created now. A chain of more than 255 codecs is rejected there too: +-- the codec count is stored in a single byte, so a longer chain used to write an unreadable part. + +-- The 104-stage chain below reserves just under 4 GiB for one block, which is what this test is +-- about, and that is most of the per-query limit CI's default profile sets. This test measures a +-- size bound, not memory consumption, so lift the limit rather than tune it. +SET max_memory_usage = 0; + +DROP TABLE IF EXISTS t_codec_chain_overflow; +DROP TABLE IF EXISTS t_codec_chain_fpc_max; +DROP TABLE IF EXISTS t_codec_chain_fpc_over; +DROP TABLE IF EXISTS t_codec_chain_gorilla_wide; +DROP TABLE IF EXISTS t_codec_chain_gorilla_max; +DROP TABLE IF EXISTS t_codec_chain_gorilla_over; +DROP TABLE IF EXISTS t_codec_chain_gorilla64_max; +DROP TABLE IF EXISTS t_codec_chain_gorilla64_over; +DROP TABLE IF EXISTS t_codec_chain_dd_max; +DROP TABLE IF EXISTS t_codec_chain_dd_over; +DROP TABLE IF EXISTS t_codec_chain_short; +DROP TABLE IF EXISTS t_codec_chain_255; +DROP TABLE IF EXISTS t_codec_chain_256; +DROP TABLE IF EXISTS t_codec_chain_256_meta; +DROP TABLE IF EXISTS t_codec_chain_meta_alter; +DROP TABLE IF EXISTS t_codec_chain_tuple_mixed; +DROP TABLE IF EXISTS t_codec_chain_tuple_wide; + +-- 150 FPC stages. Compressing a block with this chain is impossible whatever the block size, so +-- the chain is rejected when the table is created. Before the fix the table was created and the +-- first write crashed the server with an out-of-bounds heap write. +CREATE TABLE t_codec_chain_overflow (x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } + +-- 104 FPC stages: the longest chain whose reserve still fits. It must keep working and round-trip. +-- The stage the reserve wraps at is a function of the compressed block size, so the three tables +-- below pin everything that sets it: 5000 Float32 then form one 20000-byte block, wrapping at +-- stage 105. A smaller block wraps later, which would make the next arm accept its chain. +CREATE TABLE t_codec_chain_fpc_max (i UInt32, x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_fpc_max SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_fpc_max; + +-- One stage more, on the same fixture, is the first chain that is rejected. +CREATE TABLE t_codec_chain_fpc_over (i UInt32, x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_fpc_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_fpc_over; + +-- Gorilla's reserve per item is a function of the column's width, so a chain of it has a boundary +-- per width. Both boundaries are pinned below, because whichever width a process compresses first +-- is the one that used to fix the reserve for every other width, and a test cannot choose that. +CREATE TABLE t_codec_chain_gorilla_wide (d Float64 CODEC(Gorilla)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_codec_chain_gorilla_wide SELECT number / 7 FROM numbers(4); +SELECT count() FROM t_codec_chain_gorilla_wide; + +-- Gorilla reserves far more per stage than FPC, so its chains are much shorter. 29 stages is the +-- longest that still fits; it must keep working too. +CREATE TABLE t_codec_chain_gorilla_max (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla_max SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_gorilla_max; + +-- The reserve is compounded the same way whatever the codec, so the boundary below is Gorilla's own +-- and one stage past it is rejected, as for FPC above. The three arms that follow pin the sibling +-- formulas, so that rejection cannot silently narrow to FPC alone. +CREATE TABLE t_codec_chain_gorilla_over (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_gorilla_over; + +-- The same boundary for a Float64 Gorilla column: 48 stages fit, 49 do not. A chain of one width +-- must be judged on its own width, so these two keep their verdicts however the Float32 arms above +-- left the process. +CREATE TABLE t_codec_chain_gorilla64_max (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla64_max SELECT number, number / 7 FROM numbers(2500); +SELECT count(), countIf(d != i / 7) FROM t_codec_chain_gorilla64_max; + +CREATE TABLE t_codec_chain_gorilla64_over (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_gorilla64_over SELECT number, number / 7 FROM numbers(2500); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_gorilla64_over; + +-- DoubleDelta reserves more per stage still: 13 is the longest chain that fits, 14 is rejected. +CREATE TABLE t_codec_chain_dd_max (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_dd_max SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_dd_max; + +CREATE TABLE t_codec_chain_dd_over (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i + SETTINGS index_granularity = 8192, index_granularity_bytes = 0, + min_compress_block_size = 20000, max_compress_block_size = 20000; +INSERT INTO t_codec_chain_dd_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_dd_over; + +-- 255 codecs is the longest chain the one-byte count can describe. Delta does not expand, so the +-- reserve stays small and only the count matters here. +CREATE TABLE t_codec_chain_255 (i UInt32, x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD)) ENGINE = MergeTree ORDER BY i; +INSERT INTO t_codec_chain_255 SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_255; + +-- One codec more used to write a part that could not be read back. +CREATE TABLE t_codec_chain_256 (i UInt32, x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD)) ENGINE = MergeTree ORDER BY i SETTINGS allow_suspicious_codecs = 1; +INSERT INTO t_codec_chain_256 SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } +SELECT count() FROM t_codec_chain_256; + +-- A chain the one-byte count cannot describe is rejected when the metadata is created, so that it +-- cannot be persisted and replicated as a codec no replica can ever write with. This check is a +-- sanity check, so the arms above, which set allow_suspicious_codecs, still reach the write path. +CREATE TABLE t_codec_chain_256_meta (i UInt32, x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD)) ENGINE = MergeTree ORDER BY i + SETTINGS allow_suspicious_codecs = 0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_codec_chain_meta_alter (i UInt32, x Float32, w Float64) ENGINE = MergeTree ORDER BY i; +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD) + SETTINGS allow_suspicious_codecs = 0; -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_codec_chain_meta_alter ADD COLUMN y Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD) + SETTINGS allow_suspicious_codecs = 0; -- { serverError BAD_ARGUMENTS } + +-- The longest chain the count can describe is still accepted there. +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x Float32 CODEC(Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,Delta,ZSTD) + SETTINGS allow_suspicious_codecs = 0; +SELECT count() FROM t_codec_chain_meta_alter; + +-- The reserve rule applies to ALTER as well, for both forms that carry a codec. +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_codec_chain_meta_alter ADD COLUMN z Float32 CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); -- { serverError BAD_ARGUMENTS } + +-- A codec whose reserve depends on the column width resolves per substream, so a heterogeneous +-- column has one chain per substream and each of them must fit. Either element order is rejected +-- because the Float32 substream alone cannot compress a block. +CREATE TABLE t_codec_chain_tuple_mixed (x Tuple(Float32, Float64) CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_codec_chain_tuple_mixed (x Tuple(Float64, Float32) CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } + +-- The same chain on a column all of whose substreams are 8 bytes wide reserves far less and stays +-- accepted, so the check above is not a blanket rejection of the chain length. +CREATE TABLE t_codec_chain_tuple_wide (x Tuple(Float64, Float64) CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC)) ENGINE = MergeTree ORDER BY tuple(); + +-- MODIFY COLUMN may omit the type, and the codec then resolves against the type the column +-- already has, so the same chain is judged by that width and not by the null-type default. +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN x CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_codec_chain_meta_alter MODIFY COLUMN w CODEC(FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC,FPC); +SELECT count() FROM t_codec_chain_meta_alter; + +-- An ordinary short chain is unaffected. +CREATE TABLE t_codec_chain_short (i UInt32, x Float32 CODEC(Delta, ZSTD)) ENGINE = MergeTree ORDER BY i; +INSERT INTO t_codec_chain_short SELECT number, number / 7 FROM numbers(5000); +SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_short; + +DROP TABLE t_codec_chain_fpc_max; +DROP TABLE t_codec_chain_fpc_over; +DROP TABLE t_codec_chain_gorilla_wide; +DROP TABLE t_codec_chain_gorilla_max; +DROP TABLE t_codec_chain_gorilla_over; +DROP TABLE t_codec_chain_gorilla64_max; +DROP TABLE t_codec_chain_gorilla64_over; +DROP TABLE t_codec_chain_dd_max; +DROP TABLE t_codec_chain_dd_over; +DROP TABLE t_codec_chain_short; +DROP TABLE t_codec_chain_255; +DROP TABLE t_codec_chain_256; +DROP TABLE t_codec_chain_meta_alter; +DROP TABLE t_codec_chain_tuple_wide; From ec976f61688ec1dce591cd7c3ab4c2a297cccdd5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 14:42:28 +0000 Subject: [PATCH 20/46] Backport #125893 to 26.3: Fix a crash and lost rows on INSERT when a row policy of a table read by a materialized view uses `IN ` --- src/Planner/Utils.cpp | 5 +++ ...v_source_table_in_hidden_filters.reference | 5 +++ ...5402_mv_source_table_in_hidden_filters.sql | 43 +++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference create mode 100644 tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 8667a10779ec9..ed1536066e300 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -27,6 +27,7 @@ #include #include +#include #include @@ -612,6 +613,10 @@ QueryTreeNodePtr buildFilterQueryTree(ASTPtr filter_expression, if (check_access_rights) checkAccessRightsForFilter(filter_query_tree, table_expression, query_context); + /// The inner query of an ordinary view reads the table itself, not the inserted block. + if (auto view_source = query_context->getViewSource(); view_source && !query_context->isViewInnerQuery()) + replaceStorageInQueryTree(filter_query_tree, query_context, view_source); + return filter_query_tree; } diff --git a/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference new file mode 100644 index 0000000000000..a71fde4e65b88 --- /dev/null +++ b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.reference @@ -0,0 +1,5 @@ +additional_table_filters [2,3] +null_source [2,3] +policy_in_join [2,3] +policy_in_table [2,3] +view_inner [1] diff --git a/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql new file mode 100644 index 0000000000000..5ea7a7dbf9460 --- /dev/null +++ b/tests/queries/0_stateless/05402_mv_source_table_in_hidden_filters.sql @@ -0,0 +1,43 @@ +-- A row policy or `additional_table_filters` of a table read by a materialized view that names the view's source table +-- reads the inserted block, like the rest of the view's query. + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO src VALUES (1); +CREATE TABLE src_null (x UInt64) ENGINE = Null; + +CREATE TABLE child_policy (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_join (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_atf (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_null (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE child_view (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE other (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO child_policy VALUES (1), (2), (3); +INSERT INTO child_join VALUES (1), (2), (3); +INSERT INTO child_atf VALUES (1), (2), (3); +INSERT INTO child_null VALUES (1), (2), (3); +INSERT INTO child_view VALUES (1); +INSERT INTO other VALUES (1), (2), (3), (4); + +CREATE ROW POLICY OR REPLACE p_in_table ON child_policy USING x IN src TO ALL; +CREATE ROW POLICY OR REPLACE p_in_join ON child_join USING x IN (SELECT o.x FROM other AS o INNER JOIN src AS s ON o.x = s.x) TO ALL; +CREATE ROW POLICY OR REPLACE p_null ON child_null USING x IN src_null TO ALL; +CREATE ROW POLICY OR REPLACE p_view ON child_view USING x IN src TO ALL; +CREATE VIEW v AS SELECT x FROM child_view; + +CREATE TABLE dst (name String, x UInt64) ENGINE = MergeTree ORDER BY (name, x); +CREATE MATERIALIZED VIEW mv_in_table TO dst AS SELECT 'policy_in_table' AS name, c.x AS x FROM src AS s CROSS JOIN child_policy AS c; +CREATE MATERIALIZED VIEW mv_in_join TO dst AS SELECT 'policy_in_join' AS name, c.x AS x FROM src AS s CROSS JOIN child_join AS c; +CREATE MATERIALIZED VIEW mv_atf TO dst AS SELECT 'additional_table_filters' AS name, c.x AS x FROM src AS s CROSS JOIN child_atf AS c; +CREATE MATERIALIZED VIEW mv_view TO dst AS SELECT 'view_inner' AS name, c.x AS x FROM src AS s CROSS JOIN v AS c; +CREATE MATERIALIZED VIEW mv_null TO dst AS SELECT 'null_source' AS name, c.x AS x FROM src_null AS s CROSS JOIN child_null AS c; + +-- Not an asynchronous insert: its flush context has no current database. +INSERT INTO src SETTINGS async_insert = 0, additional_table_filters = {'child_atf': 'x IN src'} VALUES (2), (3), (4); +INSERT INTO src_null SETTINGS async_insert = 0 VALUES (2), (3), (4); + +SELECT name, groupArray(x) FROM (SELECT DISTINCT name, x FROM dst ORDER BY name, x) GROUP BY name ORDER BY name; + +DROP ROW POLICY p_in_table ON child_policy; +DROP ROW POLICY p_in_join ON child_join; +DROP ROW POLICY p_null ON child_null; +DROP ROW POLICY p_view ON child_view; From 4ed56374c0f427daaedb06814b837d63e870dc4c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 20:49:13 +0000 Subject: [PATCH 21/46] Backport #126550 to 26.3: Limit nesting depth of messages in Protobuf schemas --- src/Common/checkStackSize.cpp | 10 +-- src/Formats/ProtobufSerializer.cpp | 42 ++++++++++- ...5422_protobuf_deep_message_chain.reference | 5 ++ .../05422_protobuf_deep_message_chain.sh | 72 +++++++++++++++++++ ...423_stack_size_too_large_message.reference | 2 + .../05423_stack_size_too_large_message.sh | 19 +++++ 6 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference create mode 100755 tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh create mode 100644 tests/queries/0_stateless/05423_stack_size_too_large_message.reference create mode 100755 tests/queries/0_stateless/05423_stack_size_too_large_message.sh diff --git a/src/Common/checkStackSize.cpp b/src/Common/checkStackSize.cpp index 919afb215a4d5..2c3f5c2f64068 100644 --- a/src/Common/checkStackSize.cpp +++ b/src/Common/checkStackSize.cpp @@ -123,11 +123,13 @@ static NO_INLINE size_t getStackSize(void ** out_address) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Frame address is greater than stack begin address"); } -[[noreturn]] [[gnu::cold]] NO_INLINE static void throwTooDeepRecursion(const void * stack_begin, const void * frame, size_t used_stack_size, size_t total_stack_size) +/// The message is sent to the client, so it does not contain the addresses of the stack and the frame, +/// to not disclose the layout of the address space of the server. +[[noreturn]] [[gnu::cold]] NO_INLINE static void throwTooDeepRecursion(size_t used_stack_size, size_t total_stack_size) { throw DB::Exception(DB::ErrorCodes::TOO_DEEP_RECURSION, - "Stack size too large. Stack address: {}, frame address: {}, stack size: {}, maximum stack size: {}", - stack_begin, frame, used_stack_size, total_stack_size); + "Stack size too large. Stack size: {}, maximum stack size: {}", + used_stack_size, total_stack_size); } void checkStackSize() @@ -165,5 +167,5 @@ void checkStackSize() /// Just check if we have eat more than a STACK_SIZE_FREE_RATIO of stack size already. if (unlikely(stack_size > max_stack_size_allowed)) - throwTooDeepRecursion(stack_bounds.address, frame_address, stack_size, stack_bounds.max_size); + throwTooDeepRecursion(stack_size, stack_bounds.max_size); } diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 6c5d36301f22f..420d951ed7c0a 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -37,8 +37,10 @@ # include # include # include +# include # include # include +# include # include # include @@ -65,6 +67,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int ILLEGAL_COLUMN; + extern const int TOO_DEEP_RECURSION; } namespace @@ -74,6 +77,11 @@ namespace using FieldTypeId = google::protobuf::FieldDescriptor::Type; using OneofDescriptor = google::protobuf::OneofDescriptor; + /// The maximum nesting depth of messages we process. The schema is provided by the user, + /// and a chain of distinct messages, each having a field of the type of the next one, is not limited + /// by the protobuf library, while the functions below process it recursively. + constexpr size_t MAX_PROTOBUF_MESSAGE_NESTING_DEPTH = 1000; + /// Compares column's name with protobuf field's name. /// This comparison is case-insensitive and ignores the difference between '.' and '_' @@ -3396,6 +3404,18 @@ namespace bool columns_are_reordered_outside, bool check_nested_while_filling_missing_columns) { + checkStackSize(); + + if (message_nesting_depth >= MAX_PROTOBUF_MESSAGE_NESTING_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Protobuf messages are nested too deeply (more than {} levels) at the message {}", + MAX_PROTOBUF_MESSAGE_NESTING_DEPTH, + quoteString(message_descriptor.full_name())); + + ++message_nesting_depth; + SCOPE_EXIT({ --message_nesting_depth; }); + std::vector field_descs; boost::container::flat_map field_descriptors_in_use; @@ -4056,6 +4076,9 @@ namespace const ProtobufReaderOrWriter reader_or_writer; std::function get_root_desc_function; std::shared_ptr root_serializer_ptr; + + /// The current depth of recursion of `buildMessageSerializerImpl`. + size_t message_nesting_depth = 0; }; template @@ -4079,7 +4102,18 @@ namespace bool allow_repeat, std::unordered_set & unresolved_descriptors) { + checkStackSize(); + chassert(unresolved_descriptors.contains(field_descriptor)); + + /// `unresolved_descriptors` contains the chain of fields of nested messages leading to the current field. + /// It protects only from recursive types, but not from long chains of distinct messages. + if (unresolved_descriptors.size() > MAX_PROTOBUF_MESSAGE_NESTING_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Protobuf messages are nested too deeply (more than {} levels) at the field {}", + MAX_PROTOBUF_MESSAGE_NESTING_DEPTH, + quoteString(field_descriptor->full_name())); if (allow_repeat && field_descriptor->is_map()) { /// We don't add the same unresolved descriptor again since we are trying to re-resolve and put in under a Tuple @@ -4170,11 +4204,14 @@ namespace nested_field_descriptor->full_name()); } + /// Remove the descriptor on every exit, including skipping an unsupported field, + /// so that `unresolved_descriptors` contains only the chain of fields leading to the current one. + SCOPE_EXIT({ unresolved_descriptors.erase(nested_field_descriptor); }); + auto nested_name_and_type = getNameAndDataTypeFromFieldRecursive( nested_field_descriptor, skip_unsupported_fields, true, unresolved_descriptors); if (!nested_name_and_type) return std::nullopt; - unresolved_descriptors.erase(nested_field_descriptor); return NameAndTypePair{fmt::format("{}_{}", field_descriptor->name(), nested_name_and_type->name), nested_name_and_type->type}; } @@ -4191,11 +4228,12 @@ namespace "ClickHouse doesn't support type recursion ({})", message_descriptor->field(i)->full_name()); } + SCOPE_EXIT({ unresolved_descriptors.erase(message_descriptor->field(i)); }); + auto nested_name_and_type = getNameAndDataTypeFromFieldRecursive( message_descriptor->field(i), skip_unsupported_fields, true, unresolved_descriptors); if (!nested_name_and_type) continue; - unresolved_descriptors.erase(message_descriptor->field(i)); nested_types.push_back(nested_name_and_type->type); nested_names.push_back(nested_name_and_type->name); } diff --git a/tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference new file mode 100644 index 0000000000000..b5b0aff8142ad --- /dev/null +++ b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.reference @@ -0,0 +1,5 @@ +f_f_f_x UInt32 +OK +080a060a040a020801 +OK +m Tuple(\n x UInt32) diff --git a/tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh new file mode 100755 index 0000000000000..4dc2a529222e2 --- /dev/null +++ b/tests/queries/0_stateless/05422_protobuf_deep_message_chain.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A chain of distinct messages, each having a field of the type of the next one, is not limited +# by the protobuf library, but was processed recursively both by the schema inference and while +# building the serializer, so a long chain used to exhaust the thread stack. + +# Prints a schema with the messages `M0`, ..., `M`, where `M` has the field `f` of the type `M`, +# and the last one has the field `x` of the type `uint32`. +make_schema() { + python3 -c " +depth = $1 +print('syntax = \"proto3\"; ' + ' '.join('message M{}{{M{} f=1;}}'.format(i, i + 1) for i in range(depth)) + ' message M{}{{uint32 x=1;}}'.format(depth)) +" +} + +# Prints the name of a column matching the field `x` of the last message, like `f.f.f.x`. +make_column_name() { + python3 -c "print('f.' * $1 + 'x')" +} + +check_too_deep() { + local output + output=$(cat) + if grep -q -F 'TOO_DEEP_RECURSION' <<< "$output" + then + echo "OK" + else + echo "FAIL: $(head -c 1000 <<< "$output" | tr '\n' ' ')" + fi +} + +run_desc() { + ${CLICKHOUSE_LOCAL} --query " + DESC format(Protobuf, '') + SETTINGS format_schema_source = 'string', format_schema = '$(make_schema "$1")', format_schema_message_name = 'M0' + " +} + +run_write() { + ${CLICKHOUSE_LOCAL} --query " + SELECT toUInt32(1) AS \`$(make_column_name "$1")\` + FORMAT Protobuf + SETTINGS format_schema_source = 'string', format_schema = '$(make_schema "$1")', format_schema_message_name = 'M0' + " +} + +DEPTH=2000 + +# Schema inference. +run_desc 3 +run_desc $DEPTH 2>&1 | check_too_deep + +# Building the serializer for a column nested as deep as the chain. +run_write 3 | od -An -tx1 -v | tr -d ' \n' +echo +run_write $DEPTH 2>&1 | check_too_deep + +# Unsupported fields skipped during schema inference must not count as nesting levels: +# a shallow message with many skipped fields of an empty message type. +SHALLOW_SCHEMA=$(python3 -c " +print('syntax = \"proto3\"; message E {} message Inner {' + ' '.join('E e{}={};'.format(i, i + 1) for i in range(1500)) + ' uint32 x=1501;} message Root {Inner m=1;}') +") +${CLICKHOUSE_LOCAL} --query " +DESC format(Protobuf, '') +SETTINGS format_schema_source = 'string', format_schema = '$SHALLOW_SCHEMA', format_schema_message_name = 'Root', + input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference = 1 +" diff --git a/tests/queries/0_stateless/05423_stack_size_too_large_message.reference b/tests/queries/0_stateless/05423_stack_size_too_large_message.reference new file mode 100644 index 0000000000000..b261da18d51a3 --- /dev/null +++ b/tests/queries/0_stateless/05423_stack_size_too_large_message.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/05423_stack_size_too_large_message.sh b/tests/queries/0_stateless/05423_stack_size_too_large_message.sh new file mode 100755 index 0000000000000..4837cd31cfeb7 --- /dev/null +++ b/tests/queries/0_stateless/05423_stack_size_too_large_message.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The message of the `TOO_DEEP_RECURSION` exception thrown by `checkStackSize` is sent to the client, +# so it must not contain the addresses of the stack and the frame of the server. +# Two `Merge` tables reading from each other recurse until the stack check fires. + +output=$(${CLICKHOUSE_LOCAL} --query " +CREATE TABLE merge1 (x UInt64) ENGINE = Merge(currentDatabase(), '^merge\\\\d\$'); +CREATE TABLE merge2 (x UInt64) ENGINE = Merge(currentDatabase(), '^merge\\\\d\$'); +SELECT * FROM merge1; +" 2>&1) + +# `grep -c` exits with code 1 when the count is zero, so do not let it fail the test. +grep -c -F 'Stack size too large' <<< "$output" ||: +grep -c -F 'address' <<< "$output" ||: From 375626e4d99a8e6228daadde783f1d291e87b02e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 21:29:12 +0000 Subject: [PATCH 22/46] Backport #126547 to 26.3: Limit nesting depth of arrays in Redis replies --- base/poco/Redis/include/Poco/Redis/Array.h | 24 +++++++ .../Redis/include/Poco/Redis/RedisStream.h | 13 ++++ base/poco/Redis/src/Client.cpp | 15 +++- ..._redis_deeply_nested_array_reply.reference | 3 + .../05426_redis_deeply_nested_array_reply.sh | 72 +++++++++++++++++++ 5 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference create mode 100755 tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh diff --git a/base/poco/Redis/include/Poco/Redis/Array.h b/base/poco/Redis/include/Poco/Redis/Array.h index 2849829638d73..2f6a3dcf442d7 100644 --- a/base/poco/Redis/include/Poco/Redis/Array.h +++ b/base/poco/Redis/include/Poco/Redis/Array.h @@ -301,10 +301,34 @@ namespace Redis return result.str(); } + /// The maximum allowed nesting depth of arrays in a reply. + /// Replies of real Redis servers are nested only a few levels deep, + /// while unbounded nesting would lead to stack overflow in the recursive parser. + static constexpr int MAX_NESTING_DEPTH = 128; + static void read(RedisInputStream & input, Array & value) { value.clear(); + struct DepthGuard + { + explicit DepthGuard(int & depth_) : depth(depth_) + { + ++depth; + } + + ~DepthGuard() + { + --depth; + } + + int & depth; + }; + + DepthGuard depth_guard(input.arrayNestingDepth()); + if (depth_guard.depth > MAX_NESTING_DEPTH) + throw RedisException("Too deep nesting of arrays in the answer received from Redis server"); + Int64 length = NumberParser::parse64(input.getline()); if (length != -1) diff --git a/base/poco/Redis/include/Poco/Redis/RedisStream.h b/base/poco/Redis/include/Poco/Redis/RedisStream.h index 128efcfb141fa..29183988871dd 100644 --- a/base/poco/Redis/include/Poco/Redis/RedisStream.h +++ b/base/poco/Redis/include/Poco/Redis/RedisStream.h @@ -107,9 +107,22 @@ namespace Redis std::string getline(); /// Redis uses \r\n as delimiter. This getline version removes /// the \r from the result. + + int & arrayNestingDepth(); + /// Returns the current nesting depth of arrays being read from the stream. + /// Used to limit the recursion depth while parsing nested arrays. + + private: + int _arrayNestingDepth = 0; }; + inline int & RedisInputStream::arrayNestingDepth() + { + return _arrayNestingDepth; + } + + } } // namespace Poco::Redis diff --git a/base/poco/Redis/src/Client.cpp b/base/poco/Redis/src/Client.cpp index 6d28f6a29d26f..99a4ea9094bfc 100644 --- a/base/poco/Redis/src/Client.cpp +++ b/base/poco/Redis/src/Client.cpp @@ -175,10 +175,23 @@ RedisType::Ptr Client::readReply() RedisType::Ptr result = RedisType::createRedisType(c); if (result.isNull()) { + /// The stream is not at a reply boundary anymore, so the connection cannot be reused. + disconnect(); throw RedisException("Invalid Redis type returned"); } - result->read(*_input); + try + { + result->read(*_input); + } + catch (...) + { + /// A partially read reply leaves unread data in the stream. + /// If the connection were reused (e.g. returned to a pool), the next request + /// would read the remainder of this reply instead of its own one. + disconnect(); + throw; + } return result; } diff --git a/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference new file mode 100644 index 0000000000000..9591613d44cf9 --- /dev/null +++ b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.reference @@ -0,0 +1,3 @@ +OK +OK +0 diff --git a/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh new file mode 100755 index 0000000000000..3ab3dc2af914b --- /dev/null +++ b/tests/queries/0_stateless/05426_redis_deeply_nested_array_reply.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A fake Redis server replies to a command mentioning the key 'deep' with an array nested 100000 levels deep, +# and to any other command with an array of nils (one per argument except the command name). +# Parsing the deep reply must fail with an exception instead of a stack overflow, +# and the connection must not be reused: the next query over the same single-connection pool +# must receive its own reply instead of the unread remainder of the deep one. + +PORT_FILE="${CLICKHOUSE_TMP}/redis_port" +rm -f "$PORT_FILE" + +python3 -c " +import os, socket, sys, threading +deep_reply = b'*1\r\n' * 100000 + b':1\r\n' +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(('127.0.0.1', 0)) +server.listen() +server.settimeout(60) +with open(sys.argv[1] + '.tmp', 'w') as f: + f.write(str(server.getsockname()[1])) +os.rename(sys.argv[1] + '.tmp', sys.argv[1]) + +def serve(conn): + reader = conn.makefile('rb') + with conn: + try: + while True: + header = reader.readline() + if not header: + return + num_args = int(header[1:]) + args = [] + for _ in range(num_args): + length = int(reader.readline()[1:]) + args.append(reader.read(length + 2)[:-2]) + if any(b'deep' in arg for arg in args): + conn.sendall(deep_reply) + else: + conn.sendall(b'*%d\r\n' % (num_args - 1) + b'\$-1\r\n' * (num_args - 1)) + except (OSError, ValueError): + pass + +try: + while True: + conn, _ = server.accept() + threading.Thread(target=serve, args=(conn,), daemon=True).start() +except socket.timeout: + pass +" "$PORT_FILE" & +SERVER_PID=$! + +for _ in {1..100}; do + [ -f "$PORT_FILE" ] && break + sleep 0.1 +done +PORT=$(cat "$PORT_FILE") + +$CLICKHOUSE_CLIENT --query "SELECT * FROM redis('127.0.0.1:${PORT}', 'k', 'k String, v String') WHERE k = 'deep'" 2>&1 | grep -q -F 'Too deep nesting of arrays' && echo OK + +$CLICKHOUSE_CLIENT --query "CREATE TABLE redis_deep (k String, v String) ENGINE = Redis('127.0.0.1:${PORT}', 0, '', 1) PRIMARY KEY k" +$CLICKHOUSE_CLIENT --query "SELECT * FROM redis_deep WHERE k = 'deep'" 2>&1 | grep -q -F 'Too deep nesting of arrays' && echo OK +$CLICKHOUSE_CLIENT --query "SELECT count() FROM redis_deep WHERE k = 'a'" +$CLICKHOUSE_CLIENT --query "DROP TABLE redis_deep" + +kill $SERVER_PID 2>/dev/null +wait $SERVER_PID 2>/dev/null +rm -f "$PORT_FILE" From 90eb017552ef40ceb9380b3628d33d89c28d0fa0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 21:39:08 +0000 Subject: [PATCH 23/46] Backport #126159 to 26.3: Fix segfault in `MySQLDump` format on `ENUM` without values --- .../Formats/Impl/MySQLDumpRowInputFormat.cpp | 8 +++++++- ...mysqldump_enum_without_arguments.reference | 7 +++++++ .../05401_mysqldump_enum_without_arguments.sh | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference create mode 100755 tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh diff --git a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp index e74012bf72caa..96bb9b23ef3cb 100644 --- a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp @@ -223,12 +223,18 @@ static NamesAndTypesList getColumnsList(const ASTExpressionList * columns_defini /// Note: Enum would be transformed to Enum8 if number of elements is less then 128, otherwise it would be transformed to Enum16. if (type_name_upper.contains("ENUM")) { + ASTPtr arguments = data_type_node->getArguments(); + if (!arguments) + throw Exception(ErrorCodes::INCORRECT_DATA, "Type {} in MySQL dump requires a list of values", data_type_node->name); + UInt16 i = 0; - for (ASTPtr & child : data_type_node->getArguments()->children) + for (ASTPtr & child : arguments->children) { auto new_child = make_intrusive(); new_child->name = "equals"; auto * literal = child->as(); + if (!literal || literal->value.getType() != Field::Types::String) + throw Exception(ErrorCodes::INCORRECT_DATA, "Values of type {} in MySQL dump must be string literals", data_type_node->name); new_child->arguments = make_intrusive(); new_child->arguments->children.push_back(make_intrusive(literal->value.safeGet())); diff --git a/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference new file mode 100644 index 0000000000000..600e544c7f992 --- /dev/null +++ b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.reference @@ -0,0 +1,7 @@ +ENUM +INCORRECT_DATA +ENUM(UInt8) +INCORRECT_DATA +ENUM(1, 2) +INCORRECT_DATA +b Enum8(\'a\' = 1, \'b\' = 2) diff --git a/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh new file mode 100755 index 0000000000000..d8b53b38873bb --- /dev/null +++ b/tests/queries/0_stateless/05401_mysqldump_enum_without_arguments.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A MySQL dump with `ENUM` without the list of values, or with non-string values, must be rejected with an exception. + +function dump() +{ + printf 'CREATE TABLE `t1` (`x` %s NOT NULL) ENGINE=InnoDB;\n/*!40101 SET character_set_client = @saved_cs_client */;\nINSERT INTO `t1` VALUES (%s);\n' "$1" "$2" +} + +for type in 'ENUM' 'ENUM(UInt8)' 'ENUM(1, 2)'; do + echo "$type" + dump "$type" 1 | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" 2>&1 | grep -o -m1 -F 'INCORRECT_DATA' +done + +dump "ENUM('a', 'b')" "'b'" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT x, toTypeName(x) FROM table" From eead9d0de186a24691c008720d10859ed405294f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 19 Oct 2026 22:47:24 +0000 Subject: [PATCH 24/46] Backport #120776 to 26.3: Fix uninitialized offsets in Variant/Dynamic/JSON columns containing only NULLs --- src/Columns/ColumnVariant.cpp | 3 +- ...offsets_uninitialized_only_nulls.reference | 4 ++ ...riant_offsets_uninitialized_only_nulls.sql | 41 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference create mode 100644 tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index fc6157b354d7e..cf704b7ae5785 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -77,7 +77,8 @@ void ColumnVariant::constructOffsetsFromDiscriminators() /// If we have only NULLs, offsets column will not contain any real offsets. if (hasOnlyNulls()) { - offsets_data.resize(discriminators_concrete->size()); + /// Offsets of NULL rows are never read for their value, but compression reads the whole buffer. + offsets_data.resize_fill(discriminators_concrete->size(), 0); } /// If we have only one non empty variant and no NULLs, /// offsets column will contain just sequential offsets 0, 1, 2, ... diff --git a/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference new file mode 100644 index 0000000000000..4cba3ca86c550 --- /dev/null +++ b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.reference @@ -0,0 +1,4 @@ +2048 2048 +1 +2048 2048 +1 diff --git a/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql new file mode 100644 index 0000000000000..8f8092aa585bc --- /dev/null +++ b/tests/queries/0_stateless/05227_variant_offsets_uninitialized_only_nulls.sql @@ -0,0 +1,41 @@ +-- Tags: memory-engine +-- Offsets of NULL rows in a Variant column are meaningless, but ColumnVariant::compress hands the +-- whole offsets buffer to LZ4, which reads every byte. Only a sanitizer build can observe this. +-- Sizing is load bearing: ColumnVector::compress skips buffers below 4096 bytes, so the block that +-- reaches the compressing sink must keep at least 512 rows. The insert block sizes are pinned so +-- that neither randomized settings nor squashing can split or re-create the block. + +DROP TABLE IF EXISTS t_variant_src; +DROP TABLE IF EXISTS t_variant_dst; +DROP TABLE IF EXISTS t_json_src; +DROP TABLE IF EXISTS t_json_dst; + +CREATE TABLE t_variant_src (n UInt64, v Variant(UInt64, String)) ENGINE = Memory; +INSERT INTO t_variant_src SELECT number, if(number % 2 = 0, NULL, number)::Variant(UInt64, String) FROM numbers(4096) +SETTINGS max_block_size = 4096, min_insert_block_size_rows = 4096, min_insert_block_size_bytes = 0; + +CREATE TABLE t_variant_dst (v Variant(UInt64, String)) ENGINE = Memory SETTINGS compress = 1; +INSERT INTO t_variant_dst SELECT v FROM t_variant_src WHERE n % 2 = 0 +SETTINGS min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; + +SELECT count(), countIf(v IS NULL) FROM t_variant_dst; + +-- The stored table must stay far below its raw offsets buffer of total_rows * 8 bytes. If this +-- line reddens, the block no longer reaches the compressing sink and the MSan oracle above is dead. +SELECT total_bytes < total_rows * 8 FROM system.tables WHERE database = currentDatabase() AND name = 't_variant_dst'; + +CREATE TABLE t_json_src (n UInt64, j JSON) ENGINE = Memory; +INSERT INTO t_json_src SELECT number, ('{"a":' || if(number % 2 = 0, 'null', toString(number)) || '}')::JSON FROM numbers(4096) +SETTINGS max_block_size = 4096, min_insert_block_size_rows = 4096, min_insert_block_size_bytes = 0; + +CREATE TABLE t_json_dst (j JSON) ENGINE = Memory SETTINGS compress = 1; +INSERT INTO t_json_dst SELECT j FROM t_json_src WHERE n % 2 = 0 +SETTINGS min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; + +SELECT count(), countIf(j.a IS NULL) FROM t_json_dst; +SELECT total_bytes < total_rows * 8 FROM system.tables WHERE database = currentDatabase() AND name = 't_json_dst'; + +DROP TABLE t_variant_src; +DROP TABLE t_variant_dst; +DROP TABLE t_json_src; +DROP TABLE t_json_dst; From 9e15eaa9efd7dad8e7aa89c47335219ea3122ccb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 09:15:17 +0000 Subject: [PATCH 25/46] Backport #126292 to 26.3: Fix heap-use-after-free in `TablesDependencyGraph` and reject a materialized view selecting from itself --- src/Databases/DDLDependencyVisitor.cpp | 8 +-- src/Databases/DDLLoadingDependencyVisitor.cpp | 3 +- src/Databases/TablesDependencyGraph.cpp | 5 ++ .../tests/gtest_tables_dependency_graph.cpp | 19 +++++++ src/Storages/StorageMaterializedView.cpp | 22 ++++++++ ...d_view_cannot_select_from_itself.reference | 2 + ...ialized_view_cannot_select_from_itself.sql | 51 +++++++++++++++++++ 7 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference create mode 100644 tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index de7993427811d..c0e95240984c3 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -300,9 +300,10 @@ namespace } else if (functionIsInOrGlobalInOperator(function.name)) { - /// x IN table_name. + /// x IN table_name. A literal, as in x IN ('db.table'), is a set of values, not a table name. /// We set evaluate=false here because we don't want to evaluate a subquery in "x IN subquery". - addQualifiedNameFromArgument(function, 1, /* evaluate= */ false); + if (function.arguments && function.arguments->children.size() > 1 && !function.arguments->children[1]->as()) + addQualifiedNameFromArgument(function, 1, /* evaluate= */ false); } else if (function.name == "dictionary") { @@ -418,7 +419,8 @@ namespace /// We're just searching for dependencies here, it's not safe to execute subqueries now. /// Use copy of the global_context and set current database, because expressions can contain currentDatabase() function. ContextMutablePtr global_context_copy = Context::createCopy(global_context); - global_context_copy->setCurrentDatabase(current_database); + if (!current_database.empty()) + global_context_copy->setCurrentDatabase(current_database); auto evaluated = evaluateConstantExpressionOrIdentifierAsLiteral(arg, global_context_copy); const auto * literal = evaluated->as(); if (!literal || (literal->value.getType() != Field::Types::String)) diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index 28a611ac58be1..3867e5dd8da56 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -85,7 +85,8 @@ ssize_t DDLMatcherBase::getPositionOfTableNameArgumentToVisit(const ASTFunction if (!args || args->children.size() != 2) return -1; - if (args->children[1]->as()) + /// A literal, as in x IN ('db.table'), is a set of values, not a table name. + if (args->children[1]->as() || args->children[1]->as()) return -1; return 1; diff --git a/src/Databases/TablesDependencyGraph.cpp b/src/Databases/TablesDependencyGraph.cpp index 5a4579e6edc79..a77d616fa5720 100644 --- a/src/Databases/TablesDependencyGraph.cpp +++ b/src/Databases/TablesDependencyGraph.cpp @@ -93,7 +93,12 @@ size_t TablesDependencyGraph::getNumberOfTables() const void TablesDependencyGraph::addDependency(const StorageID & table_id, const StorageID & dependency) { auto * table_node = addOrUpdateNode(table_id); + + /// Adding `dependency` removes `table_node` from the graph if it gives the name of `table_node` to another node. + NodeSharedPtr table_node_holder = *nodes.find(table_node); auto * dependency_node = addOrUpdateNode(dependency); + if (!nodes.contains(table_node)) + table_node = dependency_node; /// It has the name `table_node` had, so it's the node of `table_id` now. bool inserted = table_node->dependencies.insert(dependency_node).second; if (!inserted) diff --git a/src/Databases/tests/gtest_tables_dependency_graph.cpp b/src/Databases/tests/gtest_tables_dependency_graph.cpp index eaf1e7ad9217f..b114536350cf8 100644 --- a/src/Databases/tests/gtest_tables_dependency_graph.cpp +++ b/src/Databases/tests/gtest_tables_dependency_graph.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -79,3 +80,21 @@ TEST(TablesDependencyGraph, SplitByDependencyLevelMixedCycle) EXPECT_EQ(split[1][0].getTableName(), "leaf"); EXPECT_EQ(split.back().size(), 2u); } + +/// After `RENAME` of a materialized view that selects from itself, the view graph holds the view +/// as two nodes: the old name with the UUID, and the new name. Adding the dependency of the new name +/// on the UUID gives the new name to the first node and removes the second one, the node of `table_id`. +TEST(TablesDependencyGraph, AddDependencyWhenTableNodeIsRemoved) +{ + TablesDependencyGraph graph("test"); + + UUID uuid = UUIDHelpers::generateV4(); + graph.addDependency(StorageID{"db", "mv", uuid}, StorageID{"db", "mv2"}); + graph.addDependency(StorageID{"db", "mv2"}, StorageID{"db", "mv2", uuid}); + + EXPECT_EQ(graph.getNumberOfTables(), 1u); + auto dependencies = graph.getDependencies(StorageID{"db", "mv2", uuid}); + ASSERT_EQ(dependencies.size(), 1u); + EXPECT_EQ(dependencies[0].getFullTableName(), "db.mv2"); + EXPECT_EQ(dependencies[0].uuid, uuid); +} diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index e89b29278c1c7..cd9b89b12c7b8 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -12,6 +12,8 @@ #include #include +#include +#include #include #include #include @@ -108,6 +110,17 @@ namespace if (!target_table_columns.has(column.name)) throw Exception(ErrorCodes::NO_SUCH_COLUMN_IN_TABLE, "Column {} does not exist in the materialized view's inner table", column.name); } + + bool selectReadsView(const StorageID & view_id, const ASTPtr & select_with_union, const ContextPtr & context) + { + /// Qualified as outside this session, where the name of a temporary table is a table of the view's database. + /// No current database: a recursive CTE stays unqualified. No `table_name`: it would be dropped from the result. + ASTPtr select_copy = select_with_union->clone(); + AddDefaultDatabaseVisitor(context->getGlobalContext(), view_id.database_name).visit(select_copy); + auto referenced_tables = getDependenciesFromCreateQuery( + context, /*table_name=*/{}, select_copy, /*current_database=*/"").dependencies; + return referenced_tables.contains(view_id.getQualifiedName()); + } } StorageMaterializedView::StorageMaterializedView( @@ -156,6 +169,8 @@ StorageMaterializedView::StorageMaterializedView( mv_db_context->setCurrentDatabase(table_id_.database_name); auto select = SelectQueryDescription::getSelectQueryFromASTForMatView(query.select->clone(), query.refresh_strategy != nullptr, mv_db_context); + if (mode == LoadingStrictnessLevel::CREATE && !query.refresh_strategy && selectReadsView(table_id_, select.select_query, mv_db_context)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Materialized view {} cannot select from itself", table_id_.getFullTableName()); if (select.select_table_id) { auto select_table_dependent_views = DatabaseCatalog::instance().getDependentViews(select.select_table_id); @@ -690,6 +705,13 @@ void StorageMaterializedView::alter( params.apply(new_metadata, mv_db_context); const auto & new_select = new_metadata.select; + bool modifies_query = std::ranges::any_of(params, [](const AlterCommand & command) { return command.type == AlterCommand::MODIFY_QUERY; }); + /// A refreshable view is not triggered by inserts, so its query may read the view itself. + if (modifies_query && !new_metadata.refresh) + { + if (selectReadsView(table_id, new_select.select_query, mv_db_context)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Materialized view {} cannot select from itself", table_id.getFullTableName()); + } new_metadata.setSelectQuery(new_select); /// Check the materialized view's inner table structure. diff --git a/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference new file mode 100644 index 0000000000000..ad05fc3309035 --- /dev/null +++ b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.reference @@ -0,0 +1,2 @@ +['mv'] +['mv2'] diff --git a/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql new file mode 100644 index 0000000000000..bc6867f0674d9 --- /dev/null +++ b/tests/queries/0_stateless/05404_materialized_view_cannot_select_from_itself.sql @@ -0,0 +1,51 @@ +CREATE TABLE src (c0 Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY tuple() AS SELECT c0 FROM src; +CREATE VIEW v AS SELECT c0 FROM mv; + +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM mv; -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM (SELECT c0 FROM mv); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM (SELECT c0 FROM src JOIN mv USING (c0)); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src JOIN mv USING (c0); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src JOIN (SELECT c0 FROM mv) AS t USING (c0); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM mv); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE c0 > (SELECT count() FROM mv); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src UNION ALL SELECT c0 FROM mv; -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM cluster('test_shard_localhost', currentDatabase(), 'mv')); -- { serverError BAD_ARGUMENTS } +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM v; -- { serverError INFINITE_LOOP } +-- A string literal on the right side of IN is a set of values, not a table. +ALTER TABLE mv MODIFY QUERY SELECT c0 FROM src WHERE toString(c0) IN ('v'); +-- A recursive CTE named like the view is not the view. +ALTER TABLE mv MODIFY QUERY WITH RECURSIVE mv AS (SELECT 1 AS c0 UNION ALL SELECT c0 + 1 FROM mv WHERE c0 < 3) SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM mv); +SELECT dependencies_table FROM system.tables WHERE database = currentDatabase() AND name = 'src'; +-- A temporary table named like the view is stored unqualified, so outside this session it is the view. +CREATE TEMPORARY TABLE mv (c0 Int32); +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv MODIFY QUERY SELECT c0 FROM mv; -- { serverError BAD_ARGUMENTS } +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv MODIFY QUERY WITH RECURSIVE mv AS (SELECT 1 AS c0 UNION ALL SELECT c0 + 1 FROM mv WHERE c0 < 3) SELECT c0 FROM src WHERE c0 IN (SELECT c0 FROM mv); +DROP TEMPORARY TABLE mv; +CREATE TEMPORARY TABLE mv3 (c0 Int32); +CREATE MATERIALIZED VIEW {CLICKHOUSE_DATABASE:Identifier}.mv3 ENGINE = MergeTree ORDER BY tuple() AS SELECT c0 FROM mv3; -- { serverError BAD_ARGUMENTS, UNKNOWN_TABLE } +DROP TEMPORARY TABLE mv3; +DROP TABLE v; +RENAME TABLE mv TO mv2; +ALTER TABLE mv2 MODIFY QUERY SELECT c0 FROM mv2; -- { serverError BAD_ARGUMENTS } +CREATE TEMPORARY TABLE mv2 (c0 Int32); +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv2 MODIFY QUERY SELECT c0 FROM mv2; -- { serverError BAD_ARGUMENTS } +ALTER TABLE {CLICKHOUSE_DATABASE:Identifier}.mv2 MODIFY QUERY SELECT c0 FROM src WHERE c0 IN mv2; -- { serverError BAD_ARGUMENTS } +DROP TEMPORARY TABLE mv2; +SELECT dependencies_table FROM system.tables WHERE database = currentDatabase() AND name = 'src'; + +-- A refreshable view is not triggered by inserts, so it may read itself. +CREATE TABLE dst (c0 Int32) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW rmv REFRESH EVERY 1 YEAR APPEND TO dst EMPTY AS SELECT c0 FROM src; +ALTER TABLE rmv MODIFY QUERY SELECT c0 FROM src WHERE c0 NOT IN (SELECT c0 FROM rmv); + +-- A string literal on the right side of IN in a column default is not a loading dependency either. +CREATE TABLE t1 (c0 String, c1 UInt8 DEFAULT c0 IN ('t2')) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t2 (c0 String, c1 UInt8 DEFAULT c0 IN ('t1')) ENGINE = MergeTree ORDER BY tuple(); + +DROP TABLE t2; +DROP TABLE t1; +DROP TABLE rmv; +DROP TABLE dst; +DROP TABLE mv2; +DROP TABLE src; From 70a2f1aa14ff2cd74a3a61a3342fe92225eea7d9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 09:19:52 +0000 Subject: [PATCH 26/46] Backport #103710 to 26.3: Fix flaky test_parallel_quorum_actually_parallel --- tests/integration/test_quorum_inserts_parallel/test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_quorum_inserts_parallel/test.py b/tests/integration/test_quorum_inserts_parallel/test.py index 0c9a0c1a5bf83..3631c41e5806e 100644 --- a/tests/integration/test_quorum_inserts_parallel/test.py +++ b/tests/integration/test_quorum_inserts_parallel/test.py @@ -42,9 +42,15 @@ def test_parallel_quorum_actually_parallel(started_cluster): p = Pool(10) + # The total sleep for the long insert (rows * seconds-per-row) must be larger + # than the time taken by the parallel inserts and assertions below. Otherwise + # the long insert can finish before the COUNT() == 2 assertions run, causing + # COUNT() to return 7 instead of 2. Sanitizer builds (`MSan`, `TSan`, + # `ASan + UBSan`) can slow client/server interaction by 10-20x, so use a + # generous total sleep duration (5 rows * 3s = 15s). def long_insert(node): node.query( - "INSERT INTO r SELECT number, toString(number) FROM numbers(5) where sleepEachRow(1) == 0", + "INSERT INTO r SELECT number, toString(number) FROM numbers(5) where sleepEachRow(3) == 0", settings=settings, ) From bde71434319f6ab5379ba9d4fb496695b272ffe4 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:22:32 +0000 Subject: [PATCH 27/46] Remove the dictionary-definition checks from 05212 on 26.3 The show_create and system.query_log checks test that the url of an HTTP dictionary source is masked in the dictionary definition. On master that comes from #123364, which is not in 26.3, so both checks fail here. The runtime paths changed by #120407 stay covered. Co-Authored-By: Claude Opus 5.5 --- ...5212_http_uri_credentials_masked.reference | 1 - .../05212_http_uri_credentials_masked.sh | 22 ++----------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference index 9f0c48987f44c..3d3aec4fea5a0 100644 --- a/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.reference @@ -2,7 +2,6 @@ url_function: OK masked url_status_failure: OK masked insert_presigned: OK masked dictionary_reload: OK masked -show_create: OK masked schema_inference_cache: OK masked schema_inference_error: OK masked basic_header_passthrough: OK diff --git a/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh index b0906c63fdef3..36f193af76f8b 100755 --- a/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh +++ b/tests/queries/0_stateless/05212_http_uri_credentials_masked.sh @@ -5,9 +5,8 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -# A credential embedded in an HTTP URI must never appear in an error message, in SHOW CREATE output, -# or in system.query_log. Only the masked form scheme://[HIDDEN]@host (or X-Amz-Signature=[HIDDEN] for -# a presigned URL) may be shown. +# A credential embedded in an HTTP URI must never appear in an error message. Only the masked form +# scheme://[HIDDEN]@host (or X-Amz-Signature=[HIDDEN] for a presigned URL) may be shown. PW="pwleakprobe9f2a" URI="http://leakuser:${PW}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/ping" @@ -58,23 +57,6 @@ ${CLICKHOUSE_CLIENT} --query "CREATE DICTIONARY dict_uri_leak (id UInt64, val St ${CLICKHOUSE_CLIENT} --query "SYSTEM RELOAD DICTIONARY dict_uri_leak" 2>&1 \ | grep -F 'in file/uri' | assert_shape "dictionary_reload" "$PW" "$USERINFO_SHAPE" -# 2b. SHOW CREATE must mask the password. -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DICTIONARY dict_uri_leak" 2>&1 | assert_shape "show_create" "$PW" "$USERINFO_SHAPE" - -# 3. system.query_log must store neither the query text nor the exception with the cleartext password. -# Print the offending rows rather than a count: on success this selects nothing (the reference has -# no line here), and any row printed on failure is the culprit - its query and exception show what -# leaked. The needle is split so that this checking query does not itself contain the contiguous -# secret. -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" -${CLICKHOUSE_CLIENT} --query " - SELECT type, query, exception - FROM system.query_log - WHERE event_date >= yesterday() - AND current_database = currentDatabase() - AND (query LIKE '%' || 'pwleakprobe' || '9f2a%' OR exception LIKE '%' || 'pwleakprobe' || '9f2a%') - FORMAT Vertical" - # 4. Schema inference stores the source URL in system.schema_inference_cache; the credential in its # userinfo must be masked there too. Inferring a schema from a credentialed /ping URL (which needs # no auth and whose "Ok." body is inferrable as one column) populates the cache. From f57caa30dce4ac13258441180ec07aa0c63195c7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 13:49:01 +0000 Subject: [PATCH 28/46] Update autogenerated version to 26.3.57.12 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 22f90d883026e..146a857afd48d 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54564) +SET(VERSION_REVISION 54565) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 57) -SET(VERSION_GITHASH ff8f6efef429ef44bc1ffb20ef189d164612600f) -SET(VERSION_DESCRIBE v26.3.57.1-lts) -SET(VERSION_STRING 26.3.57.1) +SET(VERSION_PATCH 58) +SET(VERSION_GITHASH ec42a9e631142812e46f558cc1c4f808831bf3a0) +SET(VERSION_DESCRIBE v26.3.58.1-lts) +SET(VERSION_STRING 26.3.58.1) # end of autochange From 548a2331755811afcb23368f07d4343bfa132e66 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 20 Oct 2026 19:29:40 +0000 Subject: [PATCH 29/46] Backport #126553 to 26.3: Fix use-after-free and reading of long values in complex key `ssd_cache` dictionaries --- src/Dictionaries/SSDCacheDictionaryStorage.h | 20 +++++++--- ...che_complex_key_reuse_partitions.reference | 4 ++ ..._ssd_cache_complex_key_reuse_partitions.sh | 37 +++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference create mode 100755 tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 0a53b2303843c..e6a732ef34a13 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -567,14 +567,16 @@ class SSDCacheFileBuffer : private boost::noncopyable return true; } - bool readKeys(size_t block_start, size_t blocks_length, PaddedPODArray & out) const + /// For complex keys, the keys written into `out` are `std::string_view`s pointing into `read_buffer_memory`, + /// so the caller must keep `read_buffer_memory` alive for as long as it uses `out`. + bool readKeys(size_t block_start, size_t blocks_length, Memory<> & read_buffer_memory, PaddedPODArray & out) const { if (block_start + blocks_length > current_blocks_size) return false; size_t buffer_size_in_bytes = blocks_length * block_size; - Memory<> read_buffer_memory(block_size * blocks_length, block_size); + read_buffer_memory = Memory<>(block_size * blocks_length, block_size); iocb request{}; iocb * request_ptr = &request; @@ -610,6 +612,9 @@ class SSDCacheFileBuffer : private boost::noncopyable throw ErrnoException(ErrorCodes::CANNOT_IO_GETEVENTS, "io_getevents: Failed to get an event for asynchronous IO"); } + // Unpoison the memory returned from an uninstrumented system function. + __msan_unpoison(&event, sizeof(event)); + auto read_bytes = eventResult(event); if (read_bytes != static_cast(buffer_size_in_bytes)) @@ -622,6 +627,9 @@ class SSDCacheFileBuffer : private boost::noncopyable ProfileEvents::increment(ProfileEvents::AIORead); ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes); + // Unpoison the memory returned from an uninstrumented system function. + __msan_unpoison(read_buffer_memory.data(), buffer_size_in_bytes); + SSDCacheBlock block(block_size); for (size_t i = 0; i < blocks_length; ++i) @@ -1124,8 +1132,8 @@ class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage for (auto & key_in_block : keys_in_block) { char * key_data = block_data + key_in_block.offset_in_block; - /// This is larger size than necessary but we don't know the exact size of serialized columns - size_t size = configuration.read_buffer_blocks_size; + /// The serialized columns are within the block, but we don't know their exact size. + size_t size = configuration.block_size - key_in_block.offset_in_block; ReadBufferFromString in({key_data, size}); deserializeAndInsertIntoColumns(result.fetched_columns, fetch_request, in); @@ -1278,8 +1286,10 @@ class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage if (started_reusing_old_partitions) { /// If we start reusing old partitions we need to remove old keys on disk from index before writing buffer + /// For complex keys, `old_keys` point into `old_keys_buffer`, so it must outlive them. + Memory<> old_keys_buffer; PaddedPODArray old_keys; - file_buffer.readKeys(block_index_in_file_before_write, configuration.write_buffer_blocks_size, old_keys); + file_buffer.readKeys(block_index_in_file_before_write, configuration.write_buffer_blocks_size, old_keys_buffer, old_keys); size_t file_read_end_block_index = block_index_in_file_before_write + configuration.write_buffer_blocks_size; diff --git a/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference new file mode 100644 index 0000000000000..388d823ad57f6 --- /dev/null +++ b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.reference @@ -0,0 +1,4 @@ +2000 2000 +2000 2000 +2000 1000 1000 +3000 diff --git a/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh new file mode 100755 index 0000000000000..2c4a7e813f02d --- /dev/null +++ b/tests/queries/0_stateless/05420_ssd_cache_complex_key_reuse_partitions.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `complex_key_ssd_cache` dictionary with tiny partitions, so that inserting keys wraps around the partitions +# and the storage reads the old keys back from the file to remove them from the index before overwriting them. +# The keys read back from the file must remain valid while they are used for the lookups in the index. +# The values read back from the file must not be truncated. + +$CLICKHOUSE_CLIENT --query=" + DROP DICTIONARY IF EXISTS ssd_cache_complex_key_reuse_dict; + DROP TABLE IF EXISTS ssd_cache_complex_key_reuse_source; + + CREATE TABLE ssd_cache_complex_key_reuse_source (k1 String, k2 UInt64, v String) ENGINE = Memory; + INSERT INTO ssd_cache_complex_key_reuse_source SELECT toString(number), number, 'value_' || toString(number) FROM numbers(3000); + + CREATE DICTIONARY ssd_cache_complex_key_reuse_dict + ( + k1 String, + k2 UInt64, + v String DEFAULT '' + ) + PRIMARY KEY k1, k2 + SOURCE(CLICKHOUSE(TABLE 'ssd_cache_complex_key_reuse_source' DB '${CLICKHOUSE_DATABASE}')) + LIFETIME(MIN 1000 MAX 1000) + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 4096 WRITE_BUFFER_SIZE 4096 MAX_PARTITIONS_COUNT 2 PATH '$USER_FILES_PATH/${CLICKHOUSE_DATABASE}_ssd_cache_complex_key_reuse')); + + SELECT count(), countIf(v = 'value_' || toString(number)) FROM (SELECT number, dictGet('ssd_cache_complex_key_reuse_dict', 'v', (toString(number), number)) AS v FROM numbers(2000)); + SELECT count(), countIf(v = 'value_' || toString(number)) FROM (SELECT number, dictGet('ssd_cache_complex_key_reuse_dict', 'v', (toString(number), number)) AS v FROM numbers(2000)); + SELECT count(), countIf(v = 'value_' || toString(number)), countIf(v = '') FROM (SELECT number, dictGet('ssd_cache_complex_key_reuse_dict', 'v', (toString(number), number)) AS v FROM numbers(2000, 2000)); + SELECT sum(dictHas('ssd_cache_complex_key_reuse_dict', (toString(number), number))) FROM numbers(4000); + + DROP DICTIONARY ssd_cache_complex_key_reuse_dict; + DROP TABLE ssd_cache_complex_key_reuse_source; +" From 0002c1817a50fa1f6535e722d07ff5ef3762bc5c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 04:47:54 +0000 Subject: [PATCH 30/46] Backport #127015 to 26.3: Fix reading uninitialized memory in `polygonConvexHullCartesian` --- src/Functions/polygonConvexHull.cpp | 19 ++++++++++++++++++- ..._convex_hull_empty_exterior_ring.reference | 6 ++++++ ...olygon_convex_hull_empty_exterior_ring.sql | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference create mode 100644 tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 7533ec5a52b6c..d678bc594305b 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -2,6 +2,7 @@ #include +#include #include namespace DB @@ -15,6 +16,21 @@ namespace ErrorCodes namespace { +/// `boost::geometry::convex_hull` takes only the exterior rings of polygons into account, +/// but it checks for emptiness with `boost::geometry::is_empty`, which also looks at the interior rings. +/// For a polygon with an empty exterior ring and a non-empty interior ring it reads uninitialized points. +/// Returns true if there are no points for the convex hull to be built from. +template +bool hasNoPointsForConvexHull(const Geometry & geometry) +{ + if constexpr (std::is_same_v, boost::geometry::polygon_tag>) + return geometry.outer().empty(); + else if constexpr (std::is_same_v, boost::geometry::multi_polygon_tag>) + return std::ranges::all_of(geometry, [](const auto & polygon) { return polygon.outer().empty(); }); + else + return boost::geometry::is_empty(geometry); +} + template class FunctionPolygonConvexHull : public IFunction { @@ -68,7 +84,8 @@ class FunctionPolygonConvexHull : public IFunction for (size_t i = 0; i < input_rows_count; ++i) { Polygon convex_hull{}; - boost::geometry::convex_hull(geometries[i], convex_hull); + if (!hasNoPointsForConvexHull(geometries[i])) + boost::geometry::convex_hull(geometries[i], convex_hull); serializer.add(convex_hull); } } diff --git a/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference new file mode 100644 index 0000000000000..0304a56da9fbe --- /dev/null +++ b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.reference @@ -0,0 +1,6 @@ +[[]] +[[]] +[[]] +POLYGON((0 0,0 5,5 5,5 0,0 0)) +[[]] +[[(0,0),(0,5),(5,5),(0,0)]] diff --git a/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql new file mode 100644 index 0000000000000..0d0036254a2a6 --- /dev/null +++ b/tests/queries/0_stateless/05427_polygon_convex_hull_empty_exterior_ring.sql @@ -0,0 +1,6 @@ +-- A polygon with an empty exterior ring and a non-empty interior ring led to reading uninitialized memory. +SELECT polygonConvexHullCartesian([[[], [(0., 0.), (1., 1.), (2., 0.)]]]::MultiPolygon); +SELECT polygonConvexHullCartesian([[], [(0., 0.), (1., 1.), (2., 0.)]]::Polygon); +SELECT polygonConvexHullCartesian([[[], [(0., 0.), (1., 1.)]], [[], [(5., 5.), (6., 6.)]]]::MultiPolygon); +SELECT wkt(polygonConvexHullCartesian([[[], [(0., 0.), (1., 1.)]], [[(0., 0.), (0., 5.), (5., 5.), (5., 0.), (2., 3.)]]]::MultiPolygon)); +SELECT polygonConvexHullCartesian(arrayJoin([[[[], [(0., 0.), (1., 1.)]]], [[[(0., 0.), (0., 5.), (5., 5.)]]]])::MultiPolygon); From 406ea17bd1bc94af5fcb33dfb88be8c088d07d42 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 14 Oct 2026 21:46:15 +0000 Subject: [PATCH 31/46] Backport #125701 to 26.8: Require READ and WRITE privileges for the FileLog table engine --- src/Storages/FileLog/StorageFileLog.cpp | 1 + ...filelog_engine_file_source_grant.reference | 6 +++ .../05394_filelog_engine_file_source_grant.sh | 40 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference create mode 100755 tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 1fd023e32e7db..19e28fe096861 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -904,6 +904,7 @@ void registerStorageFileLog(StorageFactory & factory) creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, + .source_access_type = AccessTypeObjects::Source::FILE, .has_builtin_setting_fn = FileLogSettings::hasBuiltin, }); } diff --git a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference new file mode 100644 index 0000000000000..d686d2c8371c1 --- /dev/null +++ b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.reference @@ -0,0 +1,6 @@ +-- no FILE grant: CREATE is refused, nothing is left behind +ACCESS_DENIED +0 +-- READ and WRITE on FILE: CREATE and SELECT work +owned_marker +secret_payload diff --git a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh new file mode 100755 index 0000000000000..e12f25a2608fa --- /dev/null +++ b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `FileLog` table reads files under `user_files_path`, the same data the `file` function and the +# `File` engine guard with the `FILE` source grant. The engine therefore declares `FILE` as its source, +# so `TABLE ENGINE ON FileLog` is no longer handed to everyone and a user without the source grant is +# refused at `CREATE TABLE` time instead of reading the staged files. + +# A user and a file in `user_files` are global names, and the flaky check runs several copies of one +# test at once, so they carry the pid as well as the unique name. +NAME="${CLICKHOUSE_TEST_UNIQUE_NAME}_$$" +USER="low_${NAME}" +FILE="${NAME}.csv" +TABLE="${CLICKHOUSE_DATABASE}.fl_${NAME}" + +# Match the privilege sentence rather than the engine name: the client echoes the failing query, which +# itself contains `FileLog`. +denied() { grep -qF "necessary to have the grant TABLE ENGINE ON FileLog"; } + +printf 'owned_marker\nsecret_payload\n' > "${USER_FILES_PATH:?}/${FILE}" + +${CLICKHOUSE_CLIENT} -q "DROP USER IF EXISTS ${USER}" +${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER}" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE, SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${USER}" + +echo "-- no FILE grant: CREATE is refused, nothing is left behind" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" 2>&1 | denied && echo "ACCESS_DENIED" +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name = 'fl_${NAME}'" + +echo "-- READ and WRITE on FILE: CREATE and SELECT work" +${CLICKHOUSE_CLIENT} -q "GRANT READ, WRITE ON FILE TO ${USER}" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "SELECT marker FROM ${TABLE} ORDER BY marker SETTINGS stream_like_engine_allow_direct_select = 1" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP USER ${USER}" +rm -f "${USER_FILES_PATH:?}/${FILE}" From 41e6b76d74d50bf36a17c2486cf627c26e80a29e Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Sun, 18 Oct 2026 11:33:13 +0200 Subject: [PATCH 32/46] Guard `FileLog` by the `FILE` source grant on 26.4 On 26.4 the implicit sync between `READ`/`WRITE` source grants and `TABLE ENGINE` uses the hardcoded `source_table_engines` list in `ContextAccess.cpp`, not `StorageFactory::StorageFeatures::source_access_type`. Deriving the list from `source_access_type` came with https://github.com/ClickHouse/ClickHouse/pull/100746 (26.6, backward incompatible), which is not on 26.4. So setting `source_access_type = FILE` for `FileLog` alone had no access effect here: with the default `table_engines_require_grant = false` everyone kept `TABLE ENGINE ON FileLog`, and with `true` (as in CI) `READ, WRITE ON FILE` did not imply it, which failed `05394_filelog_engine_file_source_grant`. Add `FileLog` to the list, paired with the `FILE` source. The list now carries the source explicitly because `FileLog` is not an alias of `FILE` in `AccessTypeObjects::Source`, and adding such an alias would also make `GRANT READ ON FileLog` valid, unlike master. CI report: https://d1k2gkhrlfqv31.cloudfront.net/clickhouse-test-reports-private/json.html?PR=81513&sha=9daa89c37ab4522b5c2ca4e02e5868ae5eec5f6b&name_0=BackportPR PR: https://github.com/ClickHouse/clickhouse-private/pull/81513 --- src/Access/ContextAccess.cpp | 47 +++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 87c03c7a81825..98cd4af879ed5 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -39,24 +39,27 @@ namespace ErrorCodes namespace { - const std::vector source_table_engines = { - "File", - "URL", - "Distributed", - "MongoDB", - "Redis", - "MySQL", - "PostgreSQL", - "SQLite", - "ODBC", - "JDBC", - "HDFS", - "S3", - "Hive", - "AzureBlobStorage", - "Kafka", - "NATS", - "RabbitMQ", + /// Table engines which require `READ` and `WRITE` on a source, paired with that source. + /// `FileLog` reads the same files under `user_files_path` as `File`, so it is guarded by `FILE`. + const std::vector> source_table_engines = { + {"File", AccessTypeObjects::Source::FILE}, + {"FileLog", AccessTypeObjects::Source::FILE}, + {"URL", AccessTypeObjects::Source::URL}, + {"Distributed", AccessTypeObjects::Source::REMOTE}, + {"MongoDB", AccessTypeObjects::Source::MONGO}, + {"Redis", AccessTypeObjects::Source::REDIS}, + {"MySQL", AccessTypeObjects::Source::MYSQL}, + {"PostgreSQL", AccessTypeObjects::Source::POSTGRES}, + {"SQLite", AccessTypeObjects::Source::SQLITE}, + {"ODBC", AccessTypeObjects::Source::ODBC}, + {"JDBC", AccessTypeObjects::Source::JDBC}, + {"HDFS", AccessTypeObjects::Source::HDFS}, + {"S3", AccessTypeObjects::Source::S3}, + {"Hive", AccessTypeObjects::Source::HIVE}, + {"AzureBlobStorage", AccessTypeObjects::Source::AZURE}, + {"Kafka", AccessTypeObjects::Source::KAFKA}, + {"NATS", AccessTypeObjects::Source::NATS}, + {"RabbitMQ", AccessTypeObjects::Source::RABBITMQ}, }; @@ -285,9 +288,9 @@ AccessRights ContextAccess::addImplicitAccessRights(const AccessRights & access, /// Sync SOURCE_READ/WRITE and TABLE_ENGINE, so only need to check TABLE_ENGINE later. if (access_control.doesTableEnginesRequireGrant()) { - for (const auto & table_engine : source_table_engines) + for (const auto & [table_engine, source] : source_table_engines) { - if (res.isGranted(AccessType::READ | AccessType::WRITE, AccessTypeObjects::unifySource(table_engine))) + if (res.isGranted(AccessType::READ | AccessType::WRITE, AccessTypeObjects::toStringSource(source))) res.grant(AccessType::TABLE_ENGINE, table_engine); } } @@ -295,9 +298,9 @@ AccessRights ContextAccess::addImplicitAccessRights(const AccessRights & access, { /// Add TABLE_ENGINE on * and then remove TABLE_ENGINE on particular engines. res.grant(AccessType::TABLE_ENGINE); - for (const auto & table_engine : source_table_engines) + for (const auto & [table_engine, source] : source_table_engines) { - if (!res.isGranted(AccessType::READ | AccessType::WRITE, AccessTypeObjects::unifySource(table_engine))) + if (!res.isGranted(AccessType::READ | AccessType::WRITE, AccessTypeObjects::toStringSource(source))) res.revoke(AccessType::TABLE_ENGINE, table_engine); } } From e3ee58fdd9b6163a6d8dc4a6605c71b3f1e88735 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Sun, 18 Oct 2026 19:37:37 +0200 Subject: [PATCH 33/46] Pass an absolute `FileLog` path in `05394_filelog_engine_file_source_grant` on 26.4 The test passed a bare file name to `FileLog`. Master resolves a relative `FileLog` path against `user_files_path` since https://github.com/ClickHouse/ClickHouse/pull/123397, which is not on 26.4. Here the name stays relative to the server working directory, so the `StorageFileLog` constructor threw `BAD_ARGUMENTS` "The absolute data path should be inside `user_files_path`" once the grant check passed. Use `${USER_FILES_PATH}/` like the other `FileLog` tests on 26.4. The privilege check runs before the engine is constructed, so the first half still verifies the `ACCESS_DENIED` without the `FILE` grant. CI report: https://d1k2gkhrlfqv31.cloudfront.net/clickhouse-test-reports-private/json.html?PR=81513&sha=262597882d04bb7982a1a907a5b63949a5bb33aa&name_0=BackportPR PR: https://github.com/ClickHouse/clickhouse-private/pull/81513 --- .../0_stateless/05394_filelog_engine_file_source_grant.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh index e12f25a2608fa..9c30dcfcdbff0 100755 --- a/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh +++ b/tests/queries/0_stateless/05394_filelog_engine_file_source_grant.sh @@ -13,6 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # test at once, so they carry the pid as well as the unique name. NAME="${CLICKHOUSE_TEST_UNIQUE_NAME}_$$" USER="low_${NAME}" +# On 26.4 `FileLog` does not resolve a relative path against `user_files_path`, so pass it absolute. FILE="${NAME}.csv" TABLE="${CLICKHOUSE_DATABASE}.fl_${NAME}" @@ -27,12 +28,12 @@ ${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER}" ${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE, SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${USER}" echo "-- no FILE grant: CREATE is refused, nothing is left behind" -${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" 2>&1 | denied && echo "ACCESS_DENIED" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${USER_FILES_PATH}/${FILE}', 'LineAsString')" 2>&1 | denied && echo "ACCESS_DENIED" ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name = 'fl_${NAME}'" echo "-- READ and WRITE on FILE: CREATE and SELECT work" ${CLICKHOUSE_CLIENT} -q "GRANT READ, WRITE ON FILE TO ${USER}" -${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${FILE}', 'LineAsString')" +${CLICKHOUSE_CLIENT} --user "${USER}" -q "CREATE TABLE ${TABLE} (marker String) ENGINE = FileLog('${USER_FILES_PATH}/${FILE}', 'LineAsString')" ${CLICKHOUSE_CLIENT} --user "${USER}" -q "SELECT marker FROM ${TABLE} ORDER BY marker SETTINGS stream_like_engine_allow_direct_select = 1" ${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" From 264bf9ac47a303f1aa75afe918b2900e5cdbb974 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:46:55 +0000 Subject: [PATCH 34/46] Skip codec sanity checks for a TTL recompression codec on ATTACH The codec chain checks of this backport run under `sanity_check`, which 26.3 still enables for a TTL `RECOMPRESS CODEC` when a table is attached. A table created before the upgrade with a TTL recompression chain of more than 255 codecs, or with a chain whose reserved size overflows, then fails to load, and queries to `system.tables` fail with ASYNC_LOAD_WAIT_FAILED. Master and 26.7 already skip these checks on ATTACH (e10d76dd49294, #108788); this takes only the sanity check half of that change. Co-Authored-By: Claude Opus 5.5 --- src/Storages/TTLDescription.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 799452214736a..649d0549c8ff6 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -506,9 +506,12 @@ TTLDescription TTLDescription::getTTLFromAST( } else if (ttl_element->mode == TTLMode::RECOMPRESS) { + /// Stored metadata must load, so `ATTACH` skips the codec sanity checks, as it does for column codecs. result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef()[Setting::allow_suspicious_codecs], context->getSettingsRef()[Setting::allow_experimental_codecs]); + ttl_element->recompression_codec, {}, + !is_attach && !context->getSettingsRef()[Setting::allow_suspicious_codecs], + context->getSettingsRef()[Setting::allow_experimental_codecs]); } } From 4004a8c7cef2f1306c3cace69ef9681376609312 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 11:41:08 +0000 Subject: [PATCH 35/46] Backport #126548 to 26.3: Limit recursion depth when deserializing query plans --- src/Interpreters/ActionsDAG.cpp | 32 +- src/Interpreters/ActionsDAG.h | 3 +- src/Processors/QueryPlan/QueryPlan.h | 6 +- src/Processors/QueryPlan/Serialization.cpp | 9 +- .../QueryPlan/SetsSerialization.cpp | 15 +- ..._plan_deserialize_deep_recursion.reference | 3 + ...7_query_plan_deserialize_deep_recursion.sh | 320 ++++++++++++++++++ 7 files changed, 374 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference create mode 100755 tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index c15192d1b4223..023336f502ac1 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -53,6 +54,7 @@ namespace ErrorCodes extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int BAD_ARGUMENTS; extern const int INCORRECT_DATA; + extern const int TOO_DEEP_RECURSION; } namespace @@ -3925,11 +3927,27 @@ static void serializeConstant(const IDataType & type, const IColumn & value, Wri type.getDefaultSerialization()->serializeBinary(data, 0, out, FormatSettings{}); } +/// The maximum nesting depth of lambda captures in a deserialized `ActionsDAG`. +/// The serialized data may come from a client, and the deserialization is recursive. +static constexpr size_t MAX_ACTIONS_DAG_DESERIALIZATION_DEPTH = 1000; + +static void checkActionsDAGDeserializationDepth(size_t nesting_depth) +{ + checkStackSize(); + + if (nesting_depth > MAX_ACTIONS_DAG_DESERIALIZATION_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Too deep nesting of lambda captures in deserialized ActionsDAG (more than {} levels)", + MAX_ACTIONS_DAG_DESERIALIZATION_DEPTH); +} + static MutableColumnPtr deserializeConstant( const IDataType & type, ReadBuffer & in, DeserializedSetsRegistry & registry, - const ContextPtr & context) + const ContextPtr & context, + size_t nesting_depth) { if (WhichDataType(type).isSet()) { @@ -3960,7 +3978,7 @@ static MutableColumnPtr deserializeConstant( { LambdaCapture capture; deserializeCapture(capture, in); - auto capture_dag = ActionsDAG::deserialize(in, registry, context); + auto capture_dag = ActionsDAG::deserialize(in, registry, context, nesting_depth + 1); UInt64 num_captured_columns; readVarUInt(num_captured_columns, in); @@ -3969,7 +3987,7 @@ static MutableColumnPtr deserializeConstant( for (auto & captured_column : captured_columns) { captured_column.type = decodeDataType(in); - captured_column.column = deserializeConstant(*captured_column.type, in, registry, context); + captured_column.column = deserializeConstant(*captured_column.type, in, registry, context, nesting_depth + 1); } auto function_expression = std::make_shared( @@ -4110,8 +4128,10 @@ void ActionsDAG::serialize(WriteBuffer & out, SerializedSetsRegistry & registry) writeVarUInt(node_to_id.at(output), out); } -ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context) +ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context, size_t nesting_depth) { + checkActionsDAGDeserializationDepth(nesting_depth); + size_t nodes_size; readVarUInt(nodes_size, in); @@ -4151,7 +4171,7 @@ ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & r if ((column_flags & 2) == 0) node.is_deterministic_constant = false; - node.column = deserializeConstant(*node.result_type, in, registry, context); + node.column = deserializeConstant(*node.result_type, in, registry, context, nesting_depth); } if (node.type == ActionType::INPUT) @@ -4193,7 +4213,7 @@ ActionsDAG ActionsDAG::deserialize(ReadBuffer & in, DeserializedSetsRegistry & r { LambdaCapture capture; deserializeCapture(capture, in); - auto capture_dag = ActionsDAG::deserialize(in, registry, context); + auto capture_dag = ActionsDAG::deserialize(in, registry, context, nesting_depth + 1); node.function_base = std::make_shared( std::make_shared( diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 8dd39b68f80d9..b9c9afa26b681 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -157,7 +157,8 @@ class ActionsDAG std::unordered_map getNodeToIdMap() const; void serialize(WriteBuffer & out, SerializedSetsRegistry & registry) const; - static ActionsDAG deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context); + /// `nesting_depth` is the depth of lambda captures, it is used internally to limit the recursion. + static ActionsDAG deserialize(ReadBuffer & in, DeserializedSetsRegistry & registry, const ContextPtr & context, size_t nesting_depth = 0); static Node createAlias(const Node & child, std::string alias); diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index 8397e9e3aeda0..4f5948d29694d 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -169,7 +169,11 @@ class QueryPlan static void cloneSubplanAndReplace(Node * node_to_replace, Node * subplan_root, Nodes & nodes); private: - struct SerializationFlags; + struct SerializationFlags + { + /// The nesting depth of plans for sets from subqueries, used on deserialize to limit the recursion. + size_t subquery_nesting_depth = 0; + }; void serialize(WriteBuffer & out, const SerializationFlags & flags) const; static QueryPlanAndSets deserialize(ReadBuffer & in, const ContextPtr & context, const SerializationFlags & flags); diff --git a/src/Processors/QueryPlan/Serialization.cpp b/src/Processors/QueryPlan/Serialization.cpp index 9e44401b78ccc..55194bf06ee56 100644 --- a/src/Processors/QueryPlan/Serialization.cpp +++ b/src/Processors/QueryPlan/Serialization.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -59,11 +60,6 @@ static Block deserializeHeader(ReadBuffer & in) return Block(std::move(columns)); } -/// Nothing is here for now -struct QueryPlan::SerializationFlags -{ -}; - void QueryPlan::serialize(WriteBuffer & out, size_t max_supported_version) const { UInt64 version = std::min(max_supported_version, DBMS_QUERY_PLAN_SERIALIZATION_VERSION); @@ -173,6 +169,9 @@ QueryPlanAndSets QueryPlan::deserialize(ReadBuffer & in, const ContextPtr & cont QueryPlanAndSets QueryPlan::deserialize(ReadBuffer & in, const ContextPtr & context, const SerializationFlags & flags) { + /// It is called recursively for plans of sets from subqueries. + checkStackSize(); + QueryPlanStepRegistry & step_registry = QueryPlanStepRegistry::instance(); DeserializedSetsRegistry sets_registry; diff --git a/src/Processors/QueryPlan/SetsSerialization.cpp b/src/Processors/QueryPlan/SetsSerialization.cpp index 7e245206fd0b3..832c52f6d8212 100644 --- a/src/Processors/QueryPlan/SetsSerialization.cpp +++ b/src/Processors/QueryPlan/SetsSerialization.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INCORRECT_DATA; + extern const int TOO_DEEP_RECURSION; } namespace Setting @@ -39,6 +40,10 @@ namespace Setting extern const SettingsOverflowMode transfer_overflow_mode; } +/// The maximum nesting depth of plans for sets from subqueries in a deserialized query plan. +/// The serialized plan may come from a client, and the deserialization is recursive. +static constexpr size_t MAX_SUBQUERY_PLAN_NESTING_DEPTH = 1000; + enum class SetSerializationKind : UInt8 { StorageSet = 1, @@ -192,7 +197,15 @@ QueryPlanAndSets QueryPlan::deserializeSets( } else if (kind == UInt8(SetSerializationKind::SubqueryPlan)) { - auto plan_for_set = QueryPlan::deserialize(in, context, flags); + auto nested_flags = flags; + ++nested_flags.subquery_nesting_depth; + if (nested_flags.subquery_nesting_depth > MAX_SUBQUERY_PLAN_NESTING_DEPTH) + throw Exception( + ErrorCodes::TOO_DEEP_RECURSION, + "Too deep nesting of subqueries for sets in deserialized query plan (more than {} levels)", + MAX_SUBQUERY_PLAN_NESTING_DEPTH); + + auto plan_for_set = QueryPlan::deserialize(in, context, nested_flags); res.sets_from_subquery.emplace_back(QueryPlanAndSets::SetFromSubquery{ {hash, std::move(columns)}, diff --git a/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference new file mode 100644 index 0000000000000..c214d11a9befd --- /dev/null +++ b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.reference @@ -0,0 +1,3 @@ +shallow plan: no TOO_DEEP_RECURSION +TOO_DEEP_RECURSION +server alive 1 diff --git a/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh new file mode 100755 index 0000000000000..2e7b1048f819d --- /dev/null +++ b/tests/queries/0_stateless/05427_query_plan_deserialize_deep_recursion.sh @@ -0,0 +1,320 @@ +#!/usr/bin/env bash +# Regression test for unbounded recursion in `ActionsDAG::deserialize`. +# +# A `QueryPlan` packet (native-protocol packet type 13) can carry an `ActionsDAG` with a +# lambda-capture constant whose capture DAG is itself another lambda-capture constant, and so +# on. `ActionsDAG::deserialize` / `deserializeConstant` recurse once per nesting level, so a +# client that sends a few thousand nested captures used to exhaust the server stack and take the +# whole process down with SIGSEGV. The fix caps the nesting depth and adds `checkStackSize`, +# turning the attack into a `TOO_DEEP_RECURSION` exception while the server keeps serving. +# +# The packet is only reachable for a query sent with the `QueryPlan` processing stage while the +# server has `process_query_plan_packet` enabled (see tests/config/config.d/process_query_plan_packet.xml). +# The client here speaks the raw native protocol pinned to revision 54482 so the revision-gated +# fields of the handshake are fixed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 - <<'PYEOF' +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_USER = os.environ.get("CLICKHOUSE_USER", "default") +CLICKHOUSE_PASSWORD = os.environ.get("CLICKHOUSE_PASSWORD", "") +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") + +CLIENT_NAME = "query plan deep recursion poc" +REVISION = 54482 + +# Plan serialization version gates for header fields (see Core/ProtocolDefines.h). +WITH_EXECUTION_LIMITS = 11 # max_threads + concurrency_control +WITH_STEP_VERSIONS = 19 # per-step version + +# ActionsDAG::ActionType (Interpreters/ActionsDAG.h) +ACTION_COLUMN = 1 + +# BinaryTypeIndex (DataTypes/DataTypesBinaryEncoding.h) +TYPE_UINT8 = b"\x01" +# Function(0 args) -> UInt8 : 0x24 +TYPE_FUNCTION = b"\x24\x00" + TYPE_UINT8 + +DEPTH = 5000 +SHALLOW_DEPTH = 10 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def varuint(x): + ba = bytearray() + writeVarUInt(x, ba) + return bytes(ba) + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def strbin(s): + ba = bytearray() + writeStringBinary(s, ba) + return bytes(ba) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary(CLICKHOUSE_USER, ba) + writeStringBinary(CLICKHOUSE_PASSWORD, ba) + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + readStringBinary(s) # proto_send chunked + readStringBinary(s) # proto_recv chunked + for _ in range(readVarUInt(s)): # password complexity rules + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce + while True: # server settings + if readStringBinary(s) == "": + break + readVarUInt(s) + readStringBinary(s) + query_plan_version = readVarUInt(s) # query plan serialization version + readVarUInt(s) # cluster function protocol version + return query_plan_version + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key + writeStringBinary("notchunked", ba) # proto_send chunked + writeStringBinary("notchunked", ba) # proto_recv chunked + writeVarUInt(0, ba) # parallel replicas protocol version + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary(CLICKHOUSE_USER, ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client version patch + ba.append(0) # opentelemetry: no trace id + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number + writeVarUInt(0, ba) # script line number + ba.append(0) # jwt: none + + +def sendQuery(s, query, stage): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings + writeStringBinary("", ba) # interserver externally granted roles + writeStringBinary("", ba) # interserver secret + writeVarUInt(stage, ba) # query processing stage + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator + s.sendall(ba) + + +def buildActionsDAG(depth): + """An ActionsDAG with `depth` nested lambda-capture constants. + + Each level is a single COLUMN node of a Function type; its constant is a lambda capture + whose capture DAG is the next level. The innermost level is a plain UInt8 constant, so a + correct (shallow) deserializer would terminate. + """ + # Terminal: one COLUMN node holding a UInt8 constant. + dag = bytearray() + dag += varuint(1) # nodes_size + dag += bytes([ACTION_COLUMN]) + dag += strbin("") # result_name + dag += TYPE_UINT8 # result_type + dag += varuint(0) # children + dag += bytes([1]) # column_flags: has column + dag += bytes([0]) # UInt8 constant value + dag += varuint(0) # inputs + dag += varuint(0) # outputs + + for _ in range(depth): + unit = bytearray() + unit += varuint(1) # nodes_size + unit += bytes([ACTION_COLUMN]) + unit += strbin("") # result_name + unit += TYPE_FUNCTION # result_type -> isFunction() branch in deserializeConstant + unit += varuint(0) # children + unit += bytes([1]) # column_flags: has column + # deserializeCapture + unit += strbin("") # capture.return_name + unit += TYPE_UINT8 # capture.return_type + unit += varuint(0) # num captured names + unit += varuint(0) # num captured types + unit += varuint(0) # num lambda arguments + # recursive ActionsDAG::deserialize for the capture DAG + unit += dag + unit += varuint(0) # num_captured_columns + unit += varuint(0) # inputs + unit += varuint(0) # outputs + dag = unit + + return bytes(dag) + + +def buildQueryPlanPacket(plan_version, depth): + body = bytearray() + writeVarUInt(plan_version, body) + if plan_version >= WITH_EXECUTION_LIMITS: + writeVarUInt(0, body) # max_threads + body.append(0) # concurrency_control + # Node tree, single root Expression step with no children. + writeVarUInt(0, body) # num_children + writeStringBinary("Expression", body) + if plan_version >= WITH_STEP_VERSIONS: + writeVarUInt(0, body) # step_version + writeStringBinary("", body) # step_description + writeVarUInt(0, body) # output header: 0 columns + writeVarUInt(0, body) # settings: 0 changed + # ExpressionStep::deserialize -> ActionsDAG::deserialize + body += buildActionsDAG(depth) + + packet = bytearray() + writeVarUInt(13, packet) # Client::QueryPlan + packet += body + return bytes(packet) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readStrict(s) # has_nested + return code, text + + +def sendPlan(depth): + """Sends a `QueryPlan` packet with `depth` nested lambda captures and returns the exception code and text, or None if the query is not rejected.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(60) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + plan_version = receiveHello(s) + sendAddendum(s) + # Stage 7 == QueryProcessingStage::QueryPlan: the server then expects a QueryPlan packet. + sendQuery(s, "SELECT 1", 7) + s.sendall(buildQueryPlanPacket(plan_version, depth)) + + # Only the first packet matters: either the exception or the start of the query result. + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + return readException(s) + return None, "packet {}".format(packet_type) + + +def main(): + # A shallow plan must not hit the recursion limit: it distinguishes the depth limit from rejecting any plan. + code, text = sendPlan(SHALLOW_DEPTH) + if code == 306: + print("UNEXPECTED TOO_DEEP_RECURSION for a shallow plan: {}".format(text)) + else: + print("shallow plan: no TOO_DEEP_RECURSION") + + code, text = sendPlan(DEPTH) + # 306 == TOO_DEEP_RECURSION. In sanitizer builds, `checkStackSize` may throw it before the depth limit is reached. + if code == 306: + print("TOO_DEEP_RECURSION") + else: + print("UNEXPECTED EXCEPTION {}: {}".format(code, text)) + + +main() +PYEOF + +# The server must still be alive and serving after rejecting the malicious packet. +${CLICKHOUSE_CLIENT} --query "SELECT 'server alive', 1" From a4b6e9cd07535e92506904f9b44972bb1b5f001a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 11:43:29 +0000 Subject: [PATCH 36/46] Backport #117407 to 26.3: Fix TOTP bypass in MySQL protocol authentication --- src/Access/Authentication.cpp | 23 +++ src/Access/Authentication.h | 14 ++ src/Access/IAccessStorage.cpp | 9 + src/Server/MySQLHandler.cpp | 25 ++- .../test_totp_auth/config/mysql.xml | 3 + .../test_totp_auth/config/users.xml | 20 ++ tests/integration/test_totp_auth/test_totp.py | 171 ++++++++++++++++++ 7 files changed, 256 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_totp_auth/config/mysql.xml diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index fa33d8eca45bf..055109dc596ba 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -132,6 +132,13 @@ namespace const MySQLNative41Credentials * mysql_credentials, const AuthenticationData & authentication_method) { + /// The Native41 auth response is a hash of the password alone, so it cannot carry a one-time password. + /// Accepting it would let the client skip the second factor. `MySQLHandler` switches such users + /// to the `Sha256Password` plugin, which transmits the actual password string with the one-time + /// password appended, verified via the `BasicCredentials` path. + if (authentication_method.getOneTimePassword()) + return false; + switch (authentication_method.getType()) { case AuthenticationType::PLAINTEXT_PASSWORD: @@ -416,6 +423,22 @@ Authentication::CredentialsCheckResult Authentication::areCredentialsValid( return CredentialsCheckResult::Fail; } +bool Authentication::mysqlNative41CannotVerify(const AuthenticationData & authentication_method) +{ + if (authentication_method.getOneTimePassword()) + return true; + + switch (authentication_method.getType()) + { + case AuthenticationType::SHA256_PASSWORD: + case AuthenticationType::SCRAM_SHA256_PASSWORD: + case AuthenticationType::BCRYPT_PASSWORD: + return true; + default: + return false; + } +} + bool Authentication::consumeOneTimePassword(const Credentials & credentials, const AuthenticationData & authentication_method) { const auto & otp_secret = authentication_method.getOneTimePassword(); diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 53e5bc8607750..3317943d48ba6 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -44,6 +44,20 @@ struct Authentication /// (e.g. a wrong password combined with a valid code) cannot consume the code. static bool consumeOneTimePassword(const Credentials & credentials, const AuthenticationData & authentication_method); + /** The `mysql_native_password` auth response (`MySQLNative41Credentials`) is a hash of the password alone, so it can be + * verified only against `plaintext_password` and `double_sha1_password` methods, and it cannot carry a one-time password. + * Accepting it for a user having any method for which this returns true would be wrong in two ways: + * - a method with a second factor would be accepted without the one-time password, bypassing it; + * - `IAccessStorage::authenticateImpl` fails close for ambiguous credentials: when the same password is accepted by + * several methods, the session is limited to the intersection of their `GRANTS` and expires at the earliest of + * their `VALID UNTIL`. A `sha256_password`, `scram_sha256_password` or `bcrypt_password` method cannot be matched + * by the Native41 response, so it would silently drop out of that combination, and the login would regain + * the broader rights or lifetime of the other method. + * Methods verified against an external system never take part in that combination and are not password hashes + * that a plaintext password could check better, so they do not count. + */ + static bool mysqlNative41CannotVerify(const AuthenticationData & authentication_method); + // A signaling class used to communicate requirements for credentials. template class Require : public Exception diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8ee88a5e39506..2beb46691b4b6 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -554,6 +555,14 @@ std::optional IAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); + /// `MySQLHandler` chooses the `mysql_native_password` plugin from an earlier snapshot of the user, and the user + /// can be altered while the client is answering. Re-check on the snapshot used for the authentication: if any + /// method cannot be enforced by the Native41 auth response, reject it instead of letting that method silently + /// drop out of the checks below. See `Authentication::mysqlNative41CannotVerify`. + if (typeid_cast(&credentials) + && std::ranges::any_of(user->authentication_methods, Authentication::mysqlNative41CannotVerify)) + throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials"); + bool skipped_not_allowed_authentication_methods = false; bool need_second_factor = false; diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 80c57d8427f19..7cef8126f5bcf 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -1,7 +1,11 @@ #include +#include #include +#include +#include #include +#include #include #include #include @@ -525,16 +529,19 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl { try { - const auto user_authentication_types = session->getAuthenticationTypesOrLogInFailure(user_name); - - for (const auto user_authentication_type : user_authentication_types) + /// Logs a login failure and throws if there is no such user. + session->getAuthenticationTypesOrLogInFailure(user_name); + + /// For compatibility with the JavaScript MySQL client, the `mysql_native_password` plugin (Native41) is used + /// when it can fully authenticate the user. Otherwise the `sha256_password` plugin is used: it transmits the + /// actual password string, which is then checked via the `BasicCredentials` path, the same as for the native + /// and HTTP protocols. See `Authentication::mysqlNative41CannotVerify`. + /// The user can be altered while the client is answering, so this choice is only a hint: `IAccessStorage::authenticateImpl` + /// re-checks it against the snapshot of the user it authenticates with and rejects Native41 credentials if needed. + const auto user = session->globalContext()->getAccessControl().read(user_name); + if (std::ranges::any_of(user->authentication_methods, DB::Authentication::mysqlNative41CannotVerify)) { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible - // (if password is specified using double SHA1). Otherwise, SHA256 plugin is used. - if (user_authentication_type == DB::AuthenticationType::SHA256_PASSWORD) - { - authPluginSSL(); - } + authPluginSSL(); } std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; diff --git a/tests/integration/test_totp_auth/config/mysql.xml b/tests/integration/test_totp_auth/config/mysql.xml new file mode 100644 index 0000000000000..782fbb025b115 --- /dev/null +++ b/tests/integration/test_totp_auth/config/mysql.xml @@ -0,0 +1,3 @@ + + 9004 + diff --git a/tests/integration/test_totp_auth/config/users.xml b/tests/integration/test_totp_auth/config/users.xml index 33441ee8cea8b..76101d677258d 100644 --- a/tests/integration/test_totp_auth/config/users.xml +++ b/tests/integration/test_totp_auth/config/users.xml @@ -76,6 +76,26 @@ + + aa+bb + + JBSWY3DPEHPK3PXPJBSWY3DPEHPK3PXP + 60 + 9 + SHA256 + + + + + e395796d6546b1b65db9d665cd43f0e858dd4303 + + GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ + 60 + 9 + SHA256 + + + pw diff --git a/tests/integration/test_totp_auth/test_totp.py b/tests/integration/test_totp_auth/test_totp.py index 73f52725f13c5..3276b1bd79e26 100644 --- a/tests/integration/test_totp_auth/test_totp.py +++ b/tests/integration/test_totp_auth/test_totp.py @@ -3,10 +3,12 @@ import hmac import os import re +import socket import struct import time import xml.etree.ElementTree as ET +import pymysql import pytest from helpers.cluster import ClickHouseCluster @@ -22,10 +24,15 @@ INTERACTIVE_USERS = [f"totuser_interactive_{i}" for i in range(3)] NO_PASSWORD_USERS = [f"totuser_no_password_{i}" for i in range(2)] EMPTY_PASSWORD_USERS = [f"totuser_empty_password_{i}" for i in range(2)] +MYSQL_USERS = { + "totuser_mysql_plaintext": "aa+bb", + "totuser_mysql_double_sha1": "abacaba", +} cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", + main_configs=["config/mysql.xml"], user_configs=["config/users.xml"], stay_alive=True, ) @@ -204,6 +211,170 @@ def test_interactive_totp_authentication(started_cluster): c.expect(expected_error) +def mysql_connect(user, password): + return pymysql.connections.Connection( + host=node.ip_address, + user=user, + password=password, + database="default", + port=9004, + ) + + +def test_mysql_protocol_requires_totp(started_cluster): + """The MySQL protocol must enforce TOTP: the `mysql_native_password` auth response is a hash + of the password alone and cannot carry a one-time password, so the server switches such + users to the `sha256_password` plugin and the client appends the TOTP to the password. + """ + + for user, password in MYSQL_USERS.items(): + # The correct password alone must not authenticate: the second factor is required. + # The message is intentionally generic; only the error code tells the reason. + with pytest.raises(pymysql.err.MySQLError) as exc_info: + mysql_connect(user, password) + assert exc_info.value.args[0] == 767 # REQUIRED_SECOND_FACTOR + + # A wrong one-time password must not authenticate. + with pytest.raises(pymysql.err.MySQLError, match="Authentication failed"): + mysql_connect(user, password + "+000000000") + + conn = mysql_connect(user, f"{password}+{get_otp(user)}") + try: + cursor = conn.cursor() + cursor.execute("SELECT currentUser()") + assert cursor.fetchall() == ((user,),) + finally: + conn.close() + + +def test_mysql_protocol_fail_close_on_unverifiable_methods(started_cluster): + """`IAccessStorage::authenticateImpl` fails close for ambiguous credentials: when the same + password is accepted by several methods, the session expires at the earliest of their + `VALID UNTIL` and is limited to the intersection of their `GRANTS`. The `mysql_native_password` + auth response can only be re-checked against `plaintext_password` and `double_sha1_password` + methods, so a `scram_sha256_password` or `bcrypt_password` method that narrows the session + would silently drop out of that combination. Such users are switched to the `sha256_password` + plugin, which transmits the actual password, so every method takes part in the check. + """ + + for user, other_method in ( + ("mysql_double_sha1_and_scram", "scram_sha256_password"), + ("mysql_double_sha1_and_bcrypt", "bcrypt_password"), + ): + node.query( + f"CREATE USER {user} IDENTIFIED WITH double_sha1_password BY 'pw', " + f"{other_method} BY 'pw' VALID UNTIL '2000-01-01 00:00:00'" + ) + try: + # The native protocol rejects the shared password as expired ... + assert "AUTHENTICATION_FAILED" in node.query_and_get_error( + "SELECT 1", user=user, password="pw" + ) + # ... and so must the MySQL protocol. + with pytest.raises(pymysql.err.MySQLError, match="Authentication failed"): + mysql_connect(user, "pw") + + # Without the narrowing method the user is served by `mysql_native_password` again. + node.query( + f"ALTER USER {user} IDENTIFIED WITH double_sha1_password BY 'pw'" + ) + conn = mysql_connect(user, "pw") + try: + cursor = conn.cursor() + cursor.execute("SELECT currentUser()") + assert cursor.fetchall() == ((user,),) + finally: + conn.close() + finally: + node.query(f"DROP USER IF EXISTS {user}") + + # A user with only such a method could not log in over MySQL at all before: now `sha256_password` serves it. + node.query( + "CREATE USER mysql_scram_only IDENTIFIED WITH scram_sha256_password BY 'pw'" + ) + try: + conn = mysql_connect("mysql_scram_only", "pw") + try: + cursor = conn.cursor() + cursor.execute("SELECT currentUser()") + assert cursor.fetchall() == (("mysql_scram_only",),) + finally: + conn.close() + finally: + node.query("DROP USER IF EXISTS mysql_scram_only") + + +def mysql_read_packet(sock): + def read_exactly(n): + data = b"" + while len(data) < n: + chunk = sock.recv(n - len(data)) + assert chunk, "connection closed by the server" + data += chunk + return data + + header = read_exactly(4) + payload_size = int.from_bytes(header[:3], "little") + return header[3], read_exactly(payload_size) + + +def mysql_write_packet(sock, sequence_id, payload): + sock.sendall(len(payload).to_bytes(3, "little") + bytes([sequence_id]) + payload) + + +def mysql_native_password_scramble(password, salt): + password_sha1 = hashlib.sha1(password.encode()).digest() + digest = hashlib.sha1(salt + hashlib.sha1(password_sha1).digest()).digest() + return bytes(a ^ b for a, b in zip(password_sha1, digest)) + + +def test_mysql_protocol_user_altered_during_authentication(started_cluster): + """The plugin is chosen from the user before the client answers, and the user can be altered meanwhile. + The `mysql_native_password` response must still be rejected if the user authenticated against has + a method that this plugin cannot enforce. The client is paused deterministically: it asks for + `sha256_password`, so the server sends an `AuthSwitchRequest` to `mysql_native_password` and waits. + """ + + user = "mysql_altered_during_auth" + node.query(f"CREATE USER {user} IDENTIFIED WITH double_sha1_password BY 'old'") + try: + with socket.create_connection((node.ip_address, 9004), timeout=30) as sock: + mysql_read_packet(sock) # Initial handshake. + + CLIENT_PROTOCOL_41 = 0x200 + CLIENT_SECURE_CONNECTION = 0x8000 + CLIENT_PLUGIN_AUTH = 0x80000 + capabilities = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH + handshake_response = ( + struct.pack(" Date: Wed, 21 Oct 2026 13:27:59 +0000 Subject: [PATCH 37/46] Backport #127139 to 26.3: Fix a crash with estimateCompressionRatio --- .../CompressionCodecDoubleDelta.cpp | 24 +++++++++++-------- src/Compression/CompressionCodecGorilla.cpp | 17 +++++++++---- .../04812_codec_chain_reserve_overflow.sql | 22 ++++++++--------- ...te_compression_ratio_large_block.reference | 2 ++ ...estimate_compression_ratio_large_block.sql | 5 ++++ 5 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference create mode 100644 tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 5ca6729ef6f04..f722986cac651 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -265,17 +265,15 @@ UInt32 getCompressedHeaderSize(UInt8 data_bytes_size) return items_count_size + data_bytes_size + first_delta_bytes_size; } -UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) +UInt64 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) { - const UInt32 items_count = uncompressed_size / data_bytes_size; + const UInt64 items_count = uncompressed_size / data_bytes_size; const auto double_delta_write_spec = getDeltaMaxWriteSpecByteSize(data_bytes_size); - const UInt32 max_item_size_bits = double_delta_write_spec.prefix_bits + double_delta_write_spec.data_bits; + const UInt64 max_item_size_bits = double_delta_write_spec.prefix_bits + double_delta_write_spec.data_bits; - // + 8 is to round up to next byte. - auto result = (items_count * max_item_size_bits + 7) / 8; - - return result; + // + 7 is to round up to next byte. + return (items_count * max_item_size_bits + 7) / 8; } template @@ -325,7 +323,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest) prev_value = curr_value; } - BitWriter writer(dest, getCompressedDataSize(sizeof(ValueType), source_size - sizeof(ValueType)*2)); + BitWriter writer(dest, getCompressedDataSize(sizeof(ValueType), static_cast(source_end - source))); int item = 2; for (; source < source_end; source += sizeof(ValueType), ++item) @@ -485,12 +483,18 @@ void CompressionCodecDoubleDelta::updateHash(SipHash & hash) const UInt32 CompressionCodecDoubleDelta::getMaxCompressedDataSize(UInt32 uncompressed_size) const { - const auto result = 2 // common header + const UInt64 result = 2 // common header + data_bytes_size // max bytes skipped if source is not properly aligned. + getCompressedHeaderSize(data_bytes_size) // data-specific header + getCompressedDataSize(data_bytes_size, uncompressed_size); - return result; + /// The block header stores the compressed block size as UInt32. + if (result > std::numeric_limits::max() - getHeaderSize()) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Cannot compress with DoubleDelta codec, block of {} bytes is too large: worst-case compressed size {} does not fit into 32 bits", + uncompressed_size, result); + + return static_cast(result); } UInt32 CompressionCodecDoubleDelta::doCompressData(const char * source, UInt32 source_size, char * dest) const diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index 73896def3b0a3..27a5259669b77 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace DB @@ -162,9 +163,9 @@ UInt32 getCompressedHeaderSize(UInt8 data_bytes_size) return items_count_size + data_bytes_size; } -UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) +UInt64 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) { - const UInt32 items_count = uncompressed_size / data_bytes_size; + const UInt64 items_count = uncompressed_size / data_bytes_size; /// Not static: data_bytes_size is an argument, so a static keeps the first caller's width. const auto DATA_BIT_LENGTH = getBitLengthOfLength(data_bytes_size); @@ -173,7 +174,7 @@ UInt32 getCompressedDataSize(UInt8 data_bytes_size, UInt32 uncompressed_size) // worst case (for 32-bit value): // 11 + 5 bits of leading zeroes bit-size + 5 bits of data bit-size + non-zero data bits. - const UInt32 max_item_size_bits = 2 + LEADING_ZEROES_BIT_LENGTH + DATA_BIT_LENGTH + data_bytes_size * 8; + const UInt64 max_item_size_bits = 2 + LEADING_ZEROES_BIT_LENGTH + DATA_BIT_LENGTH + data_bytes_size * 8; // + 8 is to round up to next byte. return (items_count * max_item_size_bits + 8) / 8; @@ -382,12 +383,18 @@ void CompressionCodecGorilla::updateHash(SipHash & hash) const UInt32 CompressionCodecGorilla::getMaxCompressedDataSize(UInt32 uncompressed_size) const { - const auto result = 2 // common header + const UInt64 result = 2 // common header + data_bytes_size // max bytes skipped if source is not properly aligned. + getCompressedHeaderSize(data_bytes_size) // data-specific header + getCompressedDataSize(data_bytes_size, uncompressed_size); - return result; + /// The block header stores the compressed block size as UInt32. + if (result > std::numeric_limits::max() - getHeaderSize()) + throw Exception(ErrorCodes::CANNOT_COMPRESS, + "Cannot compress with Gorilla codec, block of {} bytes is too large: worst-case compressed size {} does not fit into 32 bits", + uncompressed_size, result); + + return static_cast(result); } UInt32 CompressionCodecGorilla::doCompressData(const char * source, UInt32 source_size, char * dest) const diff --git a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql index f047a1d4dc1af..c61a43f96ce5d 100644 --- a/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql +++ b/tests/queries/0_stateless/04812_codec_chain_reserve_overflow.sql @@ -1,7 +1,7 @@ -- Tags: no-fasttest, no-parallel -- Tag no-fasttest: a codec chain long enough to overflow a 32-bit reserve needs a second or two. --- Tag no-parallel: the 104-stage arm below allocates just under 4 GiB against a per-user memory --- limit that concurrent copies of this test share, and that SET max_memory_usage = 0 does not lift. +-- Tag no-parallel: the longest accepted chains below reserve just under 4 GiB against a per-user +-- memory limit that concurrent copies of this test share, and that SET max_memory_usage = 0 does not lift. -- Random settings limits: min_columns_to_activate_adaptive_write_buffer=(3, None) -- CompressionCodecMultiple compounded each stage's reserved size in a UInt32 without checking for @@ -63,9 +63,9 @@ CREATE TABLE t_codec_chain_gorilla_wide (d Float64 CODEC(Gorilla)) ENGINE = Merg INSERT INTO t_codec_chain_gorilla_wide SELECT number / 7 FROM numbers(4); SELECT count() FROM t_codec_chain_gorilla_wide; --- Gorilla reserves far more per stage than FPC, so its chains are much shorter. 29 stages is the +-- Gorilla reserves far more per stage than FPC, so its chains are much shorter. 36 stages is the -- longest that still fits; it must keep working too. -CREATE TABLE t_codec_chain_gorilla_max (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla_max (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla_max SELECT number, number / 7 FROM numbers(5000); @@ -74,35 +74,35 @@ SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_gorilla_max; -- The reserve is compounded the same way whatever the codec, so the boundary below is Gorilla's own -- and one stage past it is rejected, as for FPC above. The three arms that follow pin the sibling -- formulas, so that rejection cannot silently narrow to FPC alone. -CREATE TABLE t_codec_chain_gorilla_over (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla_over (i UInt32, x Float32 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } SELECT count() FROM t_codec_chain_gorilla_over; --- The same boundary for a Float64 Gorilla column: 48 stages fit, 49 do not. A chain of one width +-- The same boundary for a Float64 Gorilla column: 58 stages fit, 59 do not. A chain of one width -- must be judged on its own width, so these two keep their verdicts however the Float32 arms above -- left the process. -CREATE TABLE t_codec_chain_gorilla64_max (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla64_max (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla64_max SELECT number, number / 7 FROM numbers(2500); SELECT count(), countIf(d != i / 7) FROM t_codec_chain_gorilla64_max; -CREATE TABLE t_codec_chain_gorilla64_over (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_gorilla64_over (i UInt32, d Float64 CODEC(Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla,Gorilla)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_gorilla64_over SELECT number, number / 7 FROM numbers(2500); -- { serverError CANNOT_COMPRESS } SELECT count() FROM t_codec_chain_gorilla64_over; --- DoubleDelta reserves more per stage still: 13 is the longest chain that fits, 14 is rejected. -CREATE TABLE t_codec_chain_dd_max (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i +-- DoubleDelta reserves more per stage still: 15 is the longest chain that fits, 16 is rejected. +CREATE TABLE t_codec_chain_dd_max (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_dd_max SELECT number, number / 7 FROM numbers(5000); SELECT count(), countIf(x != toFloat32(i / 7)) FROM t_codec_chain_dd_max; -CREATE TABLE t_codec_chain_dd_over (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i +CREATE TABLE t_codec_chain_dd_over (i UInt32, x Float32 CODEC(DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta,DoubleDelta)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_compress_block_size = 20000, max_compress_block_size = 20000; INSERT INTO t_codec_chain_dd_over SELECT number, number / 7 FROM numbers(5000); -- { serverError CANNOT_COMPRESS } diff --git a/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference new file mode 100644 index 0000000000000..1c2c5f0ea97f1 --- /dev/null +++ b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.reference @@ -0,0 +1,2 @@ +16 +8 diff --git a/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql new file mode 100644 index 0000000000000..014d9f2f8ce0e --- /dev/null +++ b/tests/queries/0_stateless/05407_estimate_compression_ratio_large_block.sql @@ -0,0 +1,5 @@ +-- Tags: no-sanitizers +-- no-sanitizers: too slow + +SELECT round(estimateCompressionRatio('DoubleDelta', 232160396)(toUInt16(number)), 2) FROM numbers(116080198) SETTINGS max_threads = 1, max_rows_to_read = 0; +SELECT round(estimateCompressionRatio('Gorilla', 252645136)(toUInt8(0)), 2) FROM numbers(252645136) SETTINGS max_threads = 1, max_rows_to_read = 0; From 8f6221eef2c339223475daa14552029ff60fc942 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 13:46:11 +0000 Subject: [PATCH 38/46] Update autogenerated version to 26.3.58.5 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 146a857afd48d..994902e179160 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54565) +SET(VERSION_REVISION 54566) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 58) -SET(VERSION_GITHASH ec42a9e631142812e46f558cc1c4f808831bf3a0) -SET(VERSION_DESCRIBE v26.3.58.1-lts) -SET(VERSION_STRING 26.3.58.1) +SET(VERSION_PATCH 59) +SET(VERSION_GITHASH 62a7db9a8ffe6920d4d7342a9ff710c97c91e914) +SET(VERSION_DESCRIBE v26.3.59.1-lts) +SET(VERSION_STRING 26.3.59.1) # end of autochange From 4fee975ae52dc11abc2ee654fdaababdcfc4509a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 16:26:19 +0000 Subject: [PATCH 39/46] Backport #126546 to 26.3: Validate malformed responses from HDFS DataNode and NameNode --- contrib/libhdfs3 | 2 +- contrib/libhdfs3-cmake/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index ceb428c52e6b4..28a9868bc1574 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit ceb428c52e6b4362a35ec18b69206d9bb94edce3 +Subproject commit 28a9868bc15749d899669e117e0c6eb9f5350923 diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 6757328df82f7..1c3b2bc651a7f 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -103,6 +103,8 @@ set(SRCS "${HDFS3_SOURCE_DIR}/client/StripedBlockUtil.cpp" "${HDFS3_SOURCE_DIR}/client/StripedInputStreamImpl.cpp" "${HDFS3_SOURCE_DIR}/client/StripedOutputStreamImpl.cpp" + "${HDFS3_SOURCE_DIR}/client/SlowMetrics.cpp" + "${HDFS3_SOURCE_DIR}/client/SlowNodeTracker.cpp" "${HDFS3_SOURCE_DIR}/client/SystemECPolicies.cpp" "${HDFS3_SOURCE_DIR}/client/dump.cpp" "${HDFS3_SOURCE_DIR}/client/erasure_coder.cpp" From 4fa93cc8e6d9e9955bf7001aa0e2174870860048 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 18:11:14 +0000 Subject: [PATCH 40/46] Backport #120887 to 26.3: Reject deeply nested JSON documents instead of overflowing the stack --- src/Formats/JSONExtractTree.cpp | 188 +++++++++++++----- src/Formats/JSONExtractTree.h | 5 +- src/Functions/FunctionsJSON.cpp | 23 ++- .../05218_json_type_deep_recursion.reference | 8 + .../05218_json_type_deep_recursion.sql | 51 +++++ 5 files changed, 224 insertions(+), 51 deletions(-) create mode 100644 tests/queries/0_stateless/05218_json_type_deep_recursion.reference create mode 100644 tests/queries/0_stateless/05218_json_type_deep_recursion.sql diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index 1a0e1a565e5d1..4dc4a33a4ce17 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -4,6 +4,7 @@ #include #include +#include #if USE_SIMDJSON #include #endif @@ -66,22 +67,22 @@ namespace ErrorCodes } template -void jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings) +bool jsonElementToStringImpl(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth, size_t depth) { if (element.isInt64()) { writeIntText(element.getInt64(), buf); - return; + return true; } if (element.isUInt64()) { writeIntText(element.getUInt64(), buf); - return; + return true; } if (element.isDouble()) { writeFloatText(element.getDouble(), buf); - return; + return true; } if (element.isBool()) { @@ -89,28 +90,42 @@ void jsonElementToString(const typename JSONParser::Element & element, WriteBuff writeCString("true", buf); else writeCString("false", buf); - return; + return true; } if (element.isString()) { writeJSONString(element.getString(), buf, format_settings); - return; + return true; + } + if (element.isArray() || element.isObject()) + { + if (max_depth && depth > max_depth) + { + writeCString("...", buf); + return false; + } + + /// The parsers build the document iteratively, so its nesting depth is bounded only by the + /// input. Checked in the nested branches only, to keep the scalar path free. + checkStackSize(); } if (element.isArray()) { + bool complete = true; writeChar('[', buf); bool need_comma = false; for (auto value : element.getArray()) { if (std::exchange(need_comma, true)) writeChar(',', buf); - jsonElementToString(value, buf, format_settings); + complete &= jsonElementToStringImpl(value, buf, format_settings, max_depth, depth + 1); } writeChar(']', buf); - return; + return complete; } if (element.isObject()) { + bool complete = true; writeChar('{', buf); bool need_comma = false; for (auto [key, value] : element.getObject()) @@ -119,16 +134,21 @@ void jsonElementToString(const typename JSONParser::Element & element, WriteBuff writeChar(',', buf); writeJSONString(key, buf, format_settings); writeChar(':', buf); - jsonElementToString(value, buf, format_settings); + complete &= jsonElementToStringImpl(value, buf, format_settings, max_depth, depth + 1); } writeChar('}', buf); - return; + return complete; } if (element.isNull()) - { writeCString("null", buf); - return; - } + + return true; +} + +template +bool jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth) +{ + return jsonElementToStringImpl(element, buf, format_settings, max_depth, 1); } template @@ -227,6 +247,29 @@ String jsonElementToString(const typename JSONParser::Element & element, const F return buf.str(); } +/// A message quotes untrusted input, so producing one must not recurse deeply or grow huge. +constexpr size_t MAX_DEPTH_IN_ERROR_MESSAGE = 64; + +template +String jsonElementToStringForError(const typename JSONParser::Element & element, const FormatSettings & format_settings) +{ + WriteBufferFromOwnString buf; + jsonElementToString(element, buf, format_settings, MAX_DEPTH_IN_ERROR_MESSAGE); + return buf.str(); +} + +/// Writes the element as a column's text value. A value cannot be elided like a message, so one +/// nested deeper than the limit is rejected instead. +template +bool jsonElementToStringValue(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, String & error) +{ + if (jsonElementToString(element, buf, format_settings, format_settings.json.max_depth)) + return true; + + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return false; +} + template class NumericNode : public JSONExtractTreeNode { @@ -264,7 +307,7 @@ class NumericNode : public JSONExtractTreeNode if (!tryGetNumericValueFromJSONElement(value, element, /*convert_bool_to_number=*/ true, insert_settings.allow_type_conversion, error)) { if (error.empty()) - error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToString(element, format_settings)); + error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToStringForError(element, format_settings)); return false; } @@ -321,7 +364,7 @@ class LowCardinalityNumericNode : public NumericNode if (!tryGetNumericValueFromJSONElement(value, element, /*convert_bool_to_number=*/ true, insert_settings.allow_type_conversion, error)) { if (error.empty()) - error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToString(element, format_settings)); + error = fmt::format("cannot read {} value from JSON element: {}", TypeName, jsonElementToStringForError(element, format_settings)); return false; } @@ -366,9 +409,16 @@ class StringNode : public JSONExtractTreeNode auto & col_str = assert_cast(column); auto & chars = col_str.getChars(); + size_t prev_chars_size = chars.size(); + bool written = false; { WriteBufferFromVector buf(chars, AppendModeTag()); - jsonElementToString(element, buf, format_settings); + written = jsonElementToStringValue(element, buf, format_settings, error); + } + if (!written) + { + chars.resize_assume_reserved(prev_chars_size); + return false; } col_str.getOffsets().push_back(chars.size()); } @@ -412,7 +462,11 @@ class LowCardinalityStringNode : public JSONExtractTreeNode if (!insert_settings.allow_type_conversion) return false; - auto value = jsonElementToString(element, format_settings); + WriteBufferFromOwnString buf; + if (!jsonElementToStringValue(element, buf, format_settings, error)) + return false; + + auto value = buf.str(); assert_cast(column).insertData(value.data(), value.size()); } else @@ -456,7 +510,11 @@ class FixedStringNode : public JSONExtractTreeNode { if (!insert_settings.allow_type_conversion) return false; - return checkValueSizeAndInsert(column, jsonElementToString(element, format_settings), error); + WriteBufferFromOwnString buf; + if (!jsonElementToStringValue(element, buf, format_settings, error)) + return false; + + return checkValueSizeAndInsert(column, buf.str(), error); } return checkValueSizeAndInsert(column, element.getString(), error); } @@ -507,7 +565,11 @@ class LowCardinalityFixedStringNode : public JSONExtractTreeNode { if (!insert_settings.allow_type_conversion) return false; - return checkValueSizeAndInsert(column, jsonElementToString(element, format_settings), error); + WriteBufferFromOwnString buf; + if (!jsonElementToStringValue(element, buf, format_settings, error)) + return false; + + return checkValueSizeAndInsert(column, buf.str(), error); } return checkValueSizeAndInsert(column, element.getString(), error); } @@ -561,7 +623,7 @@ class UUIDNode : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -606,7 +668,7 @@ class LowCardinalityUUIDNode : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read UUID value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -645,7 +707,7 @@ class DateNode : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read Date value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Date value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -697,7 +759,7 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi } else { - error = fmt::format("cannot read DateTime value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read DateTime value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -762,7 +824,7 @@ class TimeNode : public JSONExtractTreeNode } else { - error = fmt::format("cannot read Time value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Time value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -843,7 +905,7 @@ class DecimalNode : public JSONExtractTreeNode } default: { - error = fmt::format("cannot read Decimal value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Decimal value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } } @@ -904,7 +966,7 @@ class DateTime64Node : public JSONExtractTreeNode, public TimezoneMi value.value = element.getInt64(); break; default: - error = fmt::format("cannot read DateTime64 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read DateTime64 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } } @@ -986,7 +1048,7 @@ class Time64Node : public JSONExtractTreeNode value.value = element.getInt64(); break; default: - error = fmt::format("cannot read Time64 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Time64 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } } @@ -1080,7 +1142,7 @@ class EnumNode : public JSONExtractTreeNode return true; } - error = fmt::format("cannot read Enum value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Enum value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1110,7 +1172,7 @@ class IPv4Node : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read IPv4 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read IPv4 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1152,7 +1214,7 @@ class IPv6Node : public JSONExtractTreeNode if (!element.isString()) { - error = fmt::format("cannot read IPv6 value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read IPv6 value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1263,7 +1325,7 @@ class ArrayNode : public JSONExtractTreeNode if (!element.isArray()) { - error = fmt::format("cannot read Array value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Array value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1425,7 +1487,7 @@ class TupleNode : public JSONExtractTreeNode return were_valid_elements; } - error = fmt::format("cannot read Tuple value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Tuple value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1456,7 +1518,7 @@ class MapNode : public JSONExtractTreeNode if (!element.isObject()) { - error = fmt::format("cannot read Map value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Map value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1537,7 +1599,7 @@ class VariantNode : public JSONExtractTreeNode } } - error = fmt::format("cannot read Map value from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("cannot read Map value from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1612,13 +1674,17 @@ class DynamicNode : public JSONExtractTreeNode /// We couldn't insert element into current variants (or skipped it intentionally), /// infer ClickHouse type for this element and add it as a new variant. - auto element_type = removeNullable(elementToDataType(element, format_settings)); + auto inferred_type = elementToDataType(element, format_settings, error); + if (!inferred_type) + return false; + + auto element_type = removeNullable(inferred_type); if (!checkIfTypeIsComplete(element_type)) { error = fmt::format( "Cannot infer the type of JSON element {}, because it contains only nulls. To use String type for elements with incomplete " "type, enable setting input_format_json_infer_incomplete_types_as_strings", - jsonElementToString(element, format_settings)); + jsonElementToStringForError(element, format_settings)); return false; } @@ -1647,10 +1713,14 @@ class DynamicNode : public JSONExtractTreeNode return true; } - DataTypePtr elementToDataType(const typename JSONParser::Element & element, const FormatSettings & format_settings) const + /// Returns nullptr with the reason in `error` if the element is nested too deep. + DataTypePtr elementToDataType(const typename JSONParser::Element & element, const FormatSettings & format_settings, String & error) const { JSONInferenceInfo json_inference_info; - auto type = elementToDataTypeImpl(element, format_settings, json_inference_info); + auto type = elementToDataTypeImpl(element, format_settings, json_inference_info, 1, error); + if (!type) + return nullptr; + transformFinalInferredJSONTypeIfNeeded(type, format_settings, &json_inference_info); if (format_settings.schema_inference_make_columns_nullable && type->haveSubtypes()) type = makeNullableRecursively(type, format_settings); @@ -1658,8 +1728,18 @@ class DynamicNode : public JSONExtractTreeNode } private: - DataTypePtr elementToDataTypeImpl(const typename JSONParser::Element & element, const FormatSettings & format_settings, JSONInferenceInfo & json_inference_info) const + DataTypePtr elementToDataTypeImpl(const typename JSONParser::Element & element, const FormatSettings & format_settings, JSONInferenceInfo & json_inference_info, size_t depth, String & error) const { + /// Array nesting depth is bounded only by the input and becomes the depth of the inferred + /// type, so the limit also protects the later unguarded recursions over that type (building + /// its name, its extract tree, its column, destroying it). checkStackSize is the backstop. + checkStackSize(); + if (depth > format_settings.json.max_depth) + { + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return nullptr; + } + switch (element.type()) { case ElementType::NULL_VALUE: @@ -1701,7 +1781,12 @@ class DynamicNode : public JSONExtractTreeNode DataTypes types; types.reserve(array.size()); for (auto value : array) - types.push_back(elementToDataTypeImpl(value, format_settings, json_inference_info)); + { + auto value_type = elementToDataTypeImpl(value, format_settings, json_inference_info, depth + 1, error); + if (!value_type) + return nullptr; + types.push_back(std::move(value_type)); + } if (types.empty()) return std::make_shared(std::make_shared()); @@ -1783,7 +1868,7 @@ class ObjectJSONNode : public JSONExtractTreeNode if (!element.isObject()) { - error = fmt::format("Cannot read JSON object from JSON element: {}", jsonElementToString(element, format_settings)); + error = fmt::format("Cannot read JSON object from JSON element: {}", jsonElementToStringForError(element, format_settings)); return false; } @@ -1795,7 +1880,7 @@ class ObjectJSONNode : public JSONExtractTreeNode /// It's not optimal, but it's a price we pay for faster reading of subcolumns. std::vector> paths_and_values_for_shared_data; /// Temporary Dynamic column that will be used to create and serialize values in shared data. - if (!traverseAndInsert(column_object, element, "", insert_settings, format_settings, paths_and_values_for_shared_data, prev_size, error, true)) + if (!traverseAndInsert(column_object, element, "", insert_settings, format_settings, paths_and_values_for_shared_data, prev_size, error, true, 1)) { /// If there was an error, restore previous state. SerializationObject::restoreColumnObject(column_object, prev_size); @@ -1873,8 +1958,19 @@ class ObjectJSONNode : public JSONExtractTreeNode size_t current_size, String & error, bool is_root, + size_t depth, bool skip_typed_path_check = false) const { + /// Object nesting depth is bounded only by the input; the limit also keeps the memory spent + /// on paths, which every level appends to, from growing quadratically with the depth. + /// checkStackSize is the backstop. + checkStackSize(); + if (depth > format_settings.json.max_depth) + { + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return false; + } + if (shouldSkipPath(current_path, insert_settings)) return true; @@ -1972,7 +2068,7 @@ class ObjectJSONNode : public JSONExtractTreeNode && typed_path_nodes.contains(path) && !canParseObjectValue(typed_paths_types.at(path)); - if (!traverseAndInsert(column_object, value, path, insert_settings, format_settings, paths_and_values_for_shared_data, current_size, error, false, skip_typed)) + if (!traverseAndInsert(column_object, value, path, insert_settings, format_settings, paths_and_values_for_shared_data, current_size, error, false, depth + 1, skip_typed)) return false; } @@ -2568,16 +2664,16 @@ std::unique_ptr> buildJSONExtractTree(const Data } #if USE_SIMDJSON -template void jsonElementToString(const SimdJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +template bool jsonElementToString(const SimdJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth); template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); #endif #if USE_RAPIDJSON -template void jsonElementToString(const RapidJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +template bool jsonElementToString(const RapidJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth); template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); template bool tryGetNumericValueFromJSONElement(Float64 & value, const RapidJSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, String & error); #else -template void jsonElementToString(const DummyJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +template bool jsonElementToString(const DummyJSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth); template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); template bool tryGetNumericValueFromJSONElement(Float64 & value, const DummyJSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, String & error); template bool tryGetNumericValueFromJSONElement(Int64 & value, const DummyJSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, String & error); diff --git a/src/Formats/JSONExtractTree.h b/src/Formats/JSONExtractTree.h index fc6b7f897912d..ee4e9d835ac2d 100644 --- a/src/Formats/JSONExtractTree.h +++ b/src/Formats/JSONExtractTree.h @@ -44,8 +44,11 @@ class JSONExtractTreeNode template std::unique_ptr> buildJSONExtractTree(const DataTypePtr & type, const char * source_for_exception_message); +/// Writes the element as JSON text, descending at most `max_depth` levels and writing `...` in place +/// of deeper values. `max_depth` == 0 does not bound the descent, leaving checkStackSize as the only +/// guard. Returns false if anything was elided. template -void jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings); +bool jsonElementToString(const typename JSONParser::Element & element, WriteBuffer & buf, const FormatSettings & format_settings, size_t max_depth = 0); template bool tryGetNumericValueFromJSONElement(NumberType & value, const typename JSONParser::Element & element, bool convert_bool_to_number, bool allow_type_conversion, String & error); diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 87469a5d1934f..e256348eb5f02 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -1165,13 +1165,22 @@ class JSONExtractRawImpl static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view, const FormatSettings & format_settings, String &) + static bool insertResultToColumn(IColumn & dest, const Element & element, std::string_view, const FormatSettings & format_settings, String & error) { ColumnString & col_str = assert_cast(dest); auto & chars = col_str.getChars(); + size_t prev_chars_size = chars.size(); + bool written = false; { WriteBufferFromVector buf(chars, AppendModeTag()); - jsonElementToString(element, buf, format_settings); + written = jsonElementToString(element, buf, format_settings, format_settings.json.max_depth); + } + /// A value nested deeper than the limit cannot be rendered whole: a failure to extract it. + if (!written) + { + chars.resize_assume_reserved(prev_chars_size); + error = fmt::format("Maximum JSON depth ({}) exceeded. Consider raising input_format_json_max_depth setting.", format_settings.json.max_depth); + return false; } col_str.getOffsets().push_back(chars.size()); return true; @@ -1201,7 +1210,11 @@ class JSONExtractArrayRawImpl ColumnArray & col_res = assert_cast(dest); for (auto value : array) - JSONExtractRawImpl::insertResultToColumn(col_res.getData(), value, {}, format_settings, error); + { + /// The result keeps one element per array element, also where one cannot be rendered. + if (!JSONExtractRawImpl::insertResultToColumn(col_res.getData(), value, {}, format_settings, error)) + col_res.getData().insertDefault(); + } col_res.getOffsets().push_back(col_res.getOffsets().back() + array.size()); return true; @@ -1239,7 +1252,9 @@ class JSONExtractKeysAndValuesRawImpl for (const auto & [key, value] : object) { col_key.insertData(key.data(), key.size()); - JSONExtractRawImpl::insertResultToColumn(col_value, value, {}, format_settings, error); + /// Keep one value per key even where the value cannot be rendered. + if (!JSONExtractRawImpl::insertResultToColumn(col_value, value, {}, format_settings, error)) + col_value.insertDefault(); } col_arr.getOffsets().push_back(col_arr.getOffsets().back() + object.size()); diff --git a/tests/queries/0_stateless/05218_json_type_deep_recursion.reference b/tests/queries/0_stateless/05218_json_type_deep_recursion.reference new file mode 100644 index 0000000000000..7d4db6a5b7f06 --- /dev/null +++ b/tests/queries/0_stateless/05218_json_type_deep_recursion.reference @@ -0,0 +1,8 @@ +0 +0 +0 +['1','','2'] +[('a','1'),('b','')] +1 +{"a":{"b":[1,2,3]},"c":"42"} +{"b":[1,2,3]} diff --git a/tests/queries/0_stateless/05218_json_type_deep_recursion.sql b/tests/queries/0_stateless/05218_json_type_deep_recursion.sql new file mode 100644 index 0000000000000..5e25ac261ac43 --- /dev/null +++ b/tests/queries/0_stateless/05218_json_type_deep_recursion.sql @@ -0,0 +1,51 @@ +-- Tags: no-fasttest +-- no-fasttest: needs the RapidJSON parser, which the Fast test build does not include. + +-- Deeply nested JSON documents must be rejected instead of exhausting the thread stack. The +-- RapidJSON parser (allow_simdjson = 0) builds the document iteratively, so it survives any depth +-- and the depth reaches the consumers that walk the parsed document. +SET allow_simdjson = 0; + +-- Each case must have a single possible rejector: which of the limit and the checkStackSize +-- backstop comes first depends on the build, since frame sizes and the share of the stack the +-- backstop allows differ. + +-- Raising the limit above any reachable depth leaves the backstop as the only rejector. Nested +-- objects go through the object traversal, nested arrays through the type inference of a dynamic +-- path value. +SELECT CAST(concat(repeat('{"x":', 300000), '0', repeat('}', 300000)) AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100000000; -- { serverError TOO_DEEP_RECURSION } +SELECT CAST(concat('{"a":', repeat('[', 300000), repeat(']', 300000), '}') AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- Extraction is bounded by the limit, and a value that exceeds it is a failure to extract, which +-- yields the default like any other. All the extractors of a value as text agree on that. +SELECT length(JSONExtract(concat(repeat('[', 300000), repeat(']', 300000)), 'String')); +SELECT length(JSONExtractRaw(concat(repeat('[', 300000), repeat(']', 300000)))); +SELECT length(JSONExtractString(concat(repeat('[', 300000), repeat(']', 300000)))); +SELECT JSONExtractArrayRaw(concat('[1,', repeat('[', 300000), repeat(']', 300000), ',2]')); +SELECT JSONExtractKeysAndValuesRaw(concat('{"a":1,"b":', repeat('[', 300000), repeat(']', 300000), '}')); + +-- Raising the limit above any reachable depth leaves the backstop as the only rejector here too. +SELECT JSONExtractRaw(concat(repeat('[', 300000), repeat(']', 300000))) FORMAT Null SETTINGS input_format_json_max_depth = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- The limit governs in both directions. These depths stay shallow: a document that survives parsing +-- is walked again by consumers whose recursion this setting does not bound. +SELECT CAST(concat(repeat('{"x":', 50), '0', repeat('}', 50)) AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } +SELECT CAST(concat(repeat('{"x":', 50), '0', repeat('}', 50)) AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100; +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON) FORMAT Null SETTINGS input_format_json_max_depth = 100; + +-- A value written as text into a String column is bounded by the limit as well, and exceeding it is +-- reported the same way rather than escaping as an exception. +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON(a String)) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } +SELECT CAST(concat('{"a":', repeat('[', 50), repeat(']', 50), '}') AS JSON(a LowCardinality(String))) FORMAT Null SETTINGS input_format_json_max_depth = 20; -- { serverError INCORRECT_DATA } + +-- An element quoted in an error message is rendered shallowly, so the message stays small. +SELECT CAST(concat('{"a":', repeat('[', 200), '1', repeat(']', 200), '}') AS JSON(a UInt32)) FORMAT Null; -- { serverError INCORRECT_DATA } + +-- Exceeding the limit is a parse error, so a too-deep row can be skipped like any other malformed row. +SELECT count() FROM format(JSONEachRow, 'x JSON', concat('{"x":', repeat('{"a":', 50), '0', repeat('}', 50), '}\n{"x":{"a":42}}')) +SETTINGS input_format_json_max_depth = 20, input_format_allow_errors_num = 1; + +-- Shallow documents must still be parsed normally. +SELECT CAST('{"a" : {"b" : [1, 2, 3]}, "c" : "42"}' AS JSON); +SELECT JSONExtractRaw('{"a" : {"b" : [1, 2, 3]}}', 'a'); From 93145c3b364245cb40ff04c14f88b55d45dfe139 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 21 Oct 2026 18:31:58 +0000 Subject: [PATCH 41/46] Backport #110695 to 26.3: Combine I/O cost with selectivity in PREWHERE condition ordering --- .../MergeTree/MergeTreeWhereOptimizer.cpp | 19 +++++++++ .../MergeTree/MergeTreeWhereOptimizer.h | 12 +++++- .../ConditionSelectivityEstimator.h | 3 ++ .../0_stateless/03580_improve_prewhere.sql | 4 +- ...rewhere_statistics_cost_ordering.reference | 5 +++ ...4513_prewhere_statistics_cost_ordering.sql | 39 +++++++++++++++++ ...tatistics_ordering_compact_parts.reference | 5 +++ ...here_statistics_ordering_compact_parts.sql | 42 +++++++++++++++++++ ...ewhere_statistics_tautology_last.reference | 4 ++ ...812_prewhere_statistics_tautology_last.sql | 38 +++++++++++++++++ 10 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.reference create mode 100644 tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.sql create mode 100644 tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.reference create mode 100644 tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.sql create mode 100644 tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.reference create mode 100644 tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.sql diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 604665de2bef8..dbfc28547453d 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -107,6 +107,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( if (it != column_sizes.end()) total_size_of_queried_columns += it->second; } + + if (estimator) + total_rows = estimator->getTotalRows(); } void MergeTreeWhereOptimizer::optimize(SelectQueryInfo & select_query_info, const ContextPtr & context) const @@ -384,6 +387,22 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree pk_positions.emplace(cond.min_position_in_primary_key); } + /// Combine I/O cost with selectivity using the classic conjunctive filter ordering rule: + /// sort by cost / (1 - selectivity), i.e. cost per rejected row. + const double rejected_rows = static_cast(total_rows) - static_cast(cond.estimated_row_count); + if (total_rows == 0) + /// No statistics: fall back to pure I/O cost. + cond.cost_with_selectivity = static_cast(cond.columns_size); + else if (rejected_rows <= 0) + /// Rejects no rows, so it is useless in PREWHERE regardless of its cost: schedule it last. + cond.cost_with_selectivity = std::numeric_limits::infinity(); + else if (cond.columns_size == 0) + /// Compact parts don't track per-column compressed sizes: fall back to pure selectivity, + /// otherwise every condition collapses to cost 0 and keeps its original position. + cond.cost_with_selectivity = static_cast(cond.estimated_row_count); + else + cond.cost_with_selectivity = static_cast(cond.columns_size) / rejected_rows; + res.emplace_back(std::move(cond)); } } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index 89ba50953a9ee..bf2b97b3640d1 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -83,6 +83,12 @@ class MergeTreeWhereOptimizer : private boost::noncopyable /// the lower the better Float64 estimated_row_count = 0; + /// Combined I/O cost and selectivity score (lower is better): cost per rejected row, + /// columns_size / (total_rows - estimated_row_count). A condition that rejects no rows + /// gets +inf (scheduled last), and when per-column sizes are unavailable (columns_size=0, + /// e.g. compact parts) it falls back to estimated_row_count so selectivity ordering is kept. + double cost_with_selectivity = 0; + /// Does the condition contain primary key column? /// If so, it is better to move it further to the end of PREWHERE chain depending on minimal position in PK of any /// column in this condition because this condition have bigger chances to be already satisfied by PK analysis. @@ -93,19 +99,20 @@ class MergeTreeWhereOptimizer : private boost::noncopyable { return fmt::format( "Condition(exp:{} viable: {}, good: {}, min_position_in_primary_key: {}, estimated_row_count: {}, " - "columns_size: {}, table_columns.size: {})", + "columns_size: {}, cost_with_selectivity: {}, table_columns.size: {})", node.getColumnName(), viable, good, min_position_in_primary_key, estimated_row_count, columns_size, + cost_with_selectivity, table_columns.size()); } auto tuple() const { - return std::make_tuple(!viable, !good, -min_position_in_primary_key, estimated_row_count, columns_size, table_columns.size()); + return std::make_tuple(!viable, !good, -min_position_in_primary_key, cost_with_selectivity, table_columns.size()); } /// Is condition a better candidate for moving to PREWHERE? @@ -179,6 +186,7 @@ class MergeTreeWhereOptimizer : private boost::noncopyable LoggerPtr log; std::unordered_map column_sizes; UInt64 total_size_of_queried_columns = 0; + UInt64 total_rows = 0; }; diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h index 28f2b4b2f4442..8a72f3ad82297 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.h +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h @@ -77,6 +77,9 @@ class ConditionSelectivityEstimator : public WithContext }; using AtomMap = std::unordered_map; static const AtomMap atom_map; + + UInt64 getTotalRows() const { return total_rows; } + private: friend class ColumnStatistics; diff --git a/tests/queries/0_stateless/03580_improve_prewhere.sql b/tests/queries/0_stateless/03580_improve_prewhere.sql index 82de90ad96e1f..f4554de573a69 100644 --- a/tests/queries/0_stateless/03580_improve_prewhere.sql +++ b/tests/queries/0_stateless/03580_improve_prewhere.sql @@ -17,7 +17,9 @@ CREATE TABLE test_improve_prewhere ( value UInt32 STATISTICS(TDigest), date Date STATISTICS(CountMin), ) ENGINE = MergeTree() -ORDER BY primary_key; +ORDER BY primary_key +-- Pin compact parts (per-column sizes = 0) so PREWHERE ordering is by selectivity alone, stable under CI-randomized part-type/serialization settings. +SETTINGS min_bytes_for_wide_part = 1000000000000, min_rows_for_wide_part = 1000000000000; INSERT INTO test_improve_prewhere SELECT diff --git a/tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.reference b/tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.reference new file mode 100644 index 0000000000000..94d60bba01dae --- /dev/null +++ b/tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.reference @@ -0,0 +1,5 @@ +-- with statistics: cheap filter is placed before the expensive Map predicate +1 +-- correctness: result is the same regardless of ordering +0 +0 diff --git a/tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.sql b/tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.sql new file mode 100644 index 0000000000000..e9fdc2ecf31c8 --- /dev/null +++ b/tests/queries/0_stateless/04513_prewhere_statistics_cost_ordering.sql @@ -0,0 +1,39 @@ +-- Regression test: when statistics are enabled, the PREWHERE optimizer must +-- combine selectivity (estimated_row_count) with I/O cost (columns_size) +-- using the classic cost / (1 - selectivity) rule, not sort by selectivity alone. +-- +-- Without this fix, the auto-collected Uniq statistic on `modality` (2 distinct +-- values → 50% selectivity) makes the optimizer place the expensive Map-key +-- predicate first because its default 1% selectivity looks "more selective", +-- ignoring that the Map column is ~500x more expensive to read. + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 0; +SET optimize_move_to_prewhere = 1; +SET query_plan_optimize_prewhere = 1; +SET allow_reorder_prewhere_conditions = 1; +SET use_statistics = 1; + +DROP TABLE IF EXISTS t_prewhere_stats_cost; +CREATE TABLE t_prewhere_stats_cost (id UInt64, modality LowCardinality(String), h Map(String, String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +-- Split `modality` 50/50: the reject-count ratio is then ~2x while the Map column dwarfs the +-- scalar, so the cost gap dominates and cheap-filter-first stays stable under randomized serialization. +INSERT INTO t_prewhere_stats_cost +SELECT number, if(number % 2 = 0, 'active', ''), map('k', repeat('v', 300), 'k2', repeat('w', 300)) +FROM numbers(200000); +OPTIMIZE TABLE t_prewhere_stats_cost FINAL; + +SELECT '-- with statistics: cheap filter is placed before the expensive Map predicate'; +SELECT position(explain, 'modality') > 0 AND position(explain, 'modality') < position(explain, 'arrayElement') AS cheap_first +FROM ( + EXPLAIN actions = 1 SELECT count() FROM t_prewhere_stats_cost WHERE modality = '' AND h['k'] = 'nope' +) WHERE explain LIKE '%Prewhere filter column%'; + +SELECT '-- correctness: result is the same regardless of ordering'; +SELECT count() FROM t_prewhere_stats_cost WHERE modality = '' AND h['k'] = 'nope'; +SELECT count() FROM t_prewhere_stats_cost WHERE modality = '' AND h['k'] = 'nope' +SETTINGS allow_reorder_prewhere_conditions = 0; + +DROP TABLE t_prewhere_stats_cost; diff --git a/tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.reference b/tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.reference new file mode 100644 index 0000000000000..e03f30525645f --- /dev/null +++ b/tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.reference @@ -0,0 +1,5 @@ +-- compact part: selective condition ordered first despite zero column sizes +1 +-- correctness: result is the same regardless of ordering +10 +10 diff --git a/tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.sql b/tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.sql new file mode 100644 index 0000000000000..b66731df33eeb --- /dev/null +++ b/tests/queries/0_stateless/04811_prewhere_statistics_ordering_compact_parts.sql @@ -0,0 +1,42 @@ +-- Regression test: on compact parts the optimizer cannot read per-column compressed +-- sizes, so the PREWHERE `cost_with_selectivity` score has no I/O-cost term (columns_size +-- is 0). It must still order conditions by selectivity (estimated_row_count) instead of +-- collapsing every condition to the same cost and keeping the original WHERE order. +-- See https://github.com/ClickHouse/ClickHouse/pull/110695 + +SET enable_analyzer = 1; +SET use_statistics = 1; +SET materialize_statistics_on_insert = 1; +SET optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1; +SET allow_reorder_prewhere_conditions = 1; -- CI may inject False, preventing statistics-based reordering + +DROP TABLE IF EXISTS t_prewhere_stats_compact; + +-- No `min_bytes_for_wide_part` override, so the part is compact and per-column sizes are 0. +CREATE TABLE t_prewhere_stats_compact +( + id UInt64, + x Int64 STATISTICS(tdigest), + y Int64 STATISTICS(tdigest) +) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; + +INSERT INTO t_prewhere_stats_compact SELECT number, number, number FROM numbers(100000); + +-- `x < 10` matches 10 rows (very selective); `y < 100000` matches all rows (not selective). +-- The non-selective condition is written first in WHERE, but the optimizer must place the +-- selective `x` condition first in PREWHERE even though columns_size is 0 on the compact part. +SELECT '-- compact part: selective condition ordered first despite zero column sizes'; +SELECT position(prewhere_line, 'less(x') > 0 + AND position(prewhere_line, 'less(x') < position(prewhere_line, 'less(y') AS x_first +FROM ( + SELECT extractAll(replaceRegexpAll(explain, '__table1\.', ''), 'Prewhere filter column: ([^\n]+)')[1] AS prewhere_line FROM ( + EXPLAIN actions = 1 SELECT count() FROM t_prewhere_stats_compact WHERE y < 100000 AND x < 10 + ) WHERE explain LIKE '%Prewhere filter column%' +); + +SELECT '-- correctness: result is the same regardless of ordering'; +SELECT count() FROM t_prewhere_stats_compact WHERE y < 100000 AND x < 10; +SELECT count() FROM t_prewhere_stats_compact WHERE y < 100000 AND x < 10 +SETTINGS allow_reorder_prewhere_conditions = 0; + +DROP TABLE t_prewhere_stats_compact; diff --git a/tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.reference b/tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.reference new file mode 100644 index 0000000000000..457e2887326b4 --- /dev/null +++ b/tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.reference @@ -0,0 +1,4 @@ +-- tautology (rejects nothing) is scheduled after the selective expensive predicate +1 +-- correctness: result is unchanged +0 diff --git a/tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.sql b/tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.sql new file mode 100644 index 0000000000000..4c6f30a6b5b55 --- /dev/null +++ b/tests/queries/0_stateless/04812_prewhere_statistics_tautology_last.sql @@ -0,0 +1,38 @@ +-- Regression: a predicate that rejects no rows (estimated_row_count == total_rows) is useless in +-- PREWHERE and must be scheduled last, even when its column is cheap. Otherwise the cost-per-rejected-row +-- score gives it a finite value (columns_size) and a cheap tautology can be placed before a genuinely +-- selective but expensive predicate. +-- See https://github.com/ClickHouse/ClickHouse/pull/110695 + +SET enable_analyzer = 1; +SET use_statistics = 1; +SET materialize_statistics_on_insert = 1; +SET optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1; +SET allow_reorder_prewhere_conditions = 1; -- CI may inject False, preventing statistics-based reordering + +DROP TABLE IF EXISTS t_prewhere_tautology; + +-- Wide parts so per-column sizes feed the cost score. tdigest on `cheap` lets the optimizer see +-- `cheap < 255` as a tautology (matches all rows); `expensive = 'absent'` has no statistic, so its +-- default equality selectivity makes it the selective but expensive (large incompressible values) predicate. +CREATE TABLE t_prewhere_tautology ( + cheap UInt8 STATISTICS(tdigest), + expensive String +) ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, auto_statistics_types = ''; + +INSERT INTO t_prewhere_tautology SELECT 1, randomPrintableASCII(4000) FROM numbers(20000); + +SELECT '-- tautology (rejects nothing) is scheduled after the selective expensive predicate'; +SELECT position(prewhere_line, 'equals(expensive') > 0 + AND position(prewhere_line, 'equals(expensive') < position(prewhere_line, 'less(cheap') AS selective_first +FROM ( + SELECT extractAll(replaceRegexpAll(explain, '__table1\.', ''), 'Prewhere filter column: ([^\n]+)')[1] AS prewhere_line FROM ( + EXPLAIN actions = 1 SELECT count() FROM t_prewhere_tautology WHERE cheap < 255 AND expensive = 'absent' + ) WHERE explain LIKE '%Prewhere filter column%' +); + +SELECT '-- correctness: result is unchanged'; +SELECT count() FROM t_prewhere_tautology WHERE cheap < 255 AND expensive = 'absent'; + +DROP TABLE t_prewhere_tautology; From 2b8dbd7e07776e7cfd60f095d635652f063bba07 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 21 Oct 2026 19:29:36 +0000 Subject: [PATCH 42/46] Adapt test_mysql_protocol_fail_close_on_unverifiable_methods to 26.3 The expired-method assertions need the fail-close for ambiguous credentials of #110144, which 26.3 does not have: here a password accepted by any method logs in, on the native protocol too. Keep the part this backport changes: a user with `double_sha1_password` and a `scram_sha256_password` or `bcrypt_password` method logs in over MySQL through the `sha256_password` plugin. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_totp_auth/test_totp.py | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/integration/test_totp_auth/test_totp.py b/tests/integration/test_totp_auth/test_totp.py index 3276b1bd79e26..d682ecf948486 100644 --- a/tests/integration/test_totp_auth/test_totp.py +++ b/tests/integration/test_totp_auth/test_totp.py @@ -248,13 +248,10 @@ def test_mysql_protocol_requires_totp(started_cluster): def test_mysql_protocol_fail_close_on_unverifiable_methods(started_cluster): - """`IAccessStorage::authenticateImpl` fails close for ambiguous credentials: when the same - password is accepted by several methods, the session expires at the earliest of their - `VALID UNTIL` and is limited to the intersection of their `GRANTS`. The `mysql_native_password` - auth response can only be re-checked against `plaintext_password` and `double_sha1_password` - methods, so a `scram_sha256_password` or `bcrypt_password` method that narrows the session - would silently drop out of that combination. Such users are switched to the `sha256_password` - plugin, which transmits the actual password, so every method takes part in the check. + """The `mysql_native_password` auth response can be re-checked only against `plaintext_password` + and `double_sha1_password` methods, so `IAccessStorage::authenticateImpl` rejects it for a user + who also has a `scram_sha256_password` or `bcrypt_password` method. Such users are switched to + the `sha256_password` plugin, which transmits the actual password, so they can still log in. """ for user, other_method in ( @@ -263,21 +260,9 @@ def test_mysql_protocol_fail_close_on_unverifiable_methods(started_cluster): ): node.query( f"CREATE USER {user} IDENTIFIED WITH double_sha1_password BY 'pw', " - f"{other_method} BY 'pw' VALID UNTIL '2000-01-01 00:00:00'" + f"{other_method} BY 'pw'" ) try: - # The native protocol rejects the shared password as expired ... - assert "AUTHENTICATION_FAILED" in node.query_and_get_error( - "SELECT 1", user=user, password="pw" - ) - # ... and so must the MySQL protocol. - with pytest.raises(pymysql.err.MySQLError, match="Authentication failed"): - mysql_connect(user, "pw") - - # Without the narrowing method the user is served by `mysql_native_password` again. - node.query( - f"ALTER USER {user} IDENTIFIED WITH double_sha1_password BY 'pw'" - ) conn = mysql_connect(user, "pw") try: cursor = conn.cursor() From ab07943792c4891b25a619077cd7db434a0f1cbd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 09:08:34 +0000 Subject: [PATCH 43/46] Backport #126914 to 26.3: Do not pre-allocate memory for the declared array size when reading ZooKeeper protocol arrays --- src/Common/ZooKeeper/ZooKeeperConstants.h | 2 + src/Common/ZooKeeper/ZooKeeperIO.h | 13 +++++- src/Coordination/tests/gtest_coordination.cpp | 43 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 0706eb9f8af67..d5c5cc119fb52 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -90,6 +90,8 @@ static constexpr int32_t PASSWORD_LENGTH = 16; /// ZooKeeper has 1 MB node size and serialization limit by default, /// but it can be raised up, so we have a slightly larger limit on our side. static constexpr int32_t MAX_STRING_OR_ARRAY_SIZE = 1 << 28; /// 256 MiB +/// Upper bound on elements reserved up front when reading an array; the declared size is attacker-controlled +static constexpr int32_t MAX_ARRAY_PREALLOCATION = 4096; static constexpr int32_t DEFAULT_SESSION_TIMEOUT_MS = 30000; static constexpr int32_t DEFAULT_MIN_SESSION_TIMEOUT_MS = 10000; static constexpr int32_t DEFAULT_MAX_SESSION_TIMEOUT_MS = 100000; diff --git a/src/Common/ZooKeeper/ZooKeeperIO.h b/src/Common/ZooKeeper/ZooKeeperIO.h index 1415bee330b51..80684d0690822 100644 --- a/src/Common/ZooKeeper/ZooKeeperIO.h +++ b/src/Common/ZooKeeper/ZooKeeperIO.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -105,9 +106,17 @@ void read(std::vector & arr, ReadBuffer & in) throw Exception::fromMessage(Error::ZMARSHALLINGERROR, "Negative size while reading array from ZooKeeper"); if (size > MAX_STRING_OR_ARRAY_SIZE) throw Exception::fromMessage(Error::ZMARSHALLINGERROR, "Too large array size while reading from ZooKeeper"); - arr.resize(size); - for (auto & elem : arr) + + /// Memory must track bytes actually read, not the declared element count: + /// do not pre-allocate for an attacker-controlled size. + arr.clear(); + arr.reserve(std::min(static_cast(size), static_cast(MAX_ARRAY_PREALLOCATION))); + for (int32_t i = 0; i < size; ++i) + { + T elem{}; read(elem, in); + arr.push_back(std::move(elem)); + } } } diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 38c9f3650e565..291bc4236055c 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -129,6 +129,49 @@ TYPED_TEST(CoordinationTest, BufferSerde) } } +TYPED_TEST(CoordinationTest, CreateRequestDoesNotPreallocateOversizedACLs) +{ + /// Invariant: memory must track bytes actually read, not the declared element count. + { + Coordination::ACLs acls; + acls.push_back(Coordination::ACL{Coordination::ACL::All, "world", "anyone"}); + acls.push_back(Coordination::ACL{Coordination::ACL::Read, "digest", "user:hash"}); + + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(std::string{"/node"}, wbuf); + Coordination::write(std::string{"data"}, wbuf); + Coordination::write(acls, wbuf); + Coordination::write(static_cast(0), wbuf); + + auto request_read = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Create); + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + request_read->readImpl(rbuf); + auto & create_read = dynamic_cast(*request_read); + ASSERT_EQ(create_read.acls.size(), 2); + EXPECT_EQ(create_read.acls[0].permissions, Coordination::ACL::All); + EXPECT_EQ(create_read.acls[0].scheme, "world"); + EXPECT_EQ(create_read.acls[0].id, "anyone"); + EXPECT_EQ(create_read.acls[1].permissions, Coordination::ACL::Read); + EXPECT_EQ(create_read.acls[1].scheme, "digest"); + EXPECT_EQ(create_read.acls[1].id, "user:hash"); + } + + { + /// Declared ACL count is above the pre-allocation bound, but no element bytes follow. + /// The reservation must stay bounded by the cap, not by the declared count. + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(std::string{"/node"}, wbuf); + Coordination::write(std::string{}, wbuf); + Coordination::write(static_cast(2 * Coordination::MAX_ARRAY_PREALLOCATION), wbuf); + + auto request_read = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Create); + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + EXPECT_THROW(request_read->readImpl(rbuf), DB::Exception); + auto & create_read = dynamic_cast(*request_read); + EXPECT_LE(create_read.acls.capacity(), static_cast(Coordination::MAX_ARRAY_PREALLOCATION)); + } +} + template struct SimpliestRaftServer { From 85b96c66ae897c8e044c0e3f4e420d5e49b0ee0a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 19:54:30 +0000 Subject: [PATCH 44/46] Update autogenerated version to 26.3.59.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 994902e179160..5b892ffc6661f 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54566) +SET(VERSION_REVISION 54567) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 59) -SET(VERSION_GITHASH 62a7db9a8ffe6920d4d7342a9ff710c97c91e914) -SET(VERSION_DESCRIBE v26.3.59.1-lts) -SET(VERSION_STRING 26.3.59.1) +SET(VERSION_PATCH 60) +SET(VERSION_GITHASH 4f0c8c19d3057d8b683a5fd87c244b1c83e8c1d7) +SET(VERSION_DESCRIBE v26.3.60.1-lts) +SET(VERSION_STRING 26.3.60.1) # end of autochange From fb7f61070281d6fbe5f7a969dd42229d243cac6a Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:17:57 +0100 Subject: [PATCH 45/46] Resolve conflicts in merge of v26.3.60.2-lts Took upstream's 26.3.60 patch and hash in Altinity's version format, and kept both upstream's gcp_oauth check and Altinity's shared_cache lookup in getClient. --- cmake/autogenerated_versions.txt | 11 ++--------- .../ObjectStorages/S3/diskSettings.cpp | 6 ++---- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 31f83e41573f5..a09ea8493e9cd 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -5,17 +5,10 @@ SET(VERSION_REVISION 54567) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -<<<<<<< HEAD -SET(VERSION_PATCH 53) -SET(VERSION_GITHASH fe8ff5d492ab7325e974b19a619b5c3ab3c095d8) -SET(VERSION_DESCRIBE v26.3.53.20001.altinityantalya) -SET(VERSION_STRING 26.3.53.20001.altinityantalya) -======= SET(VERSION_PATCH 60) SET(VERSION_GITHASH 4f0c8c19d3057d8b683a5fd87c244b1c83e8c1d7) -SET(VERSION_DESCRIBE v26.3.60.1-lts) -SET(VERSION_STRING 26.3.60.1) ->>>>>>> v26.3.60.2-lts +SET(VERSION_DESCRIBE v26.3.60.20001.altinityantalya) +SET(VERSION_STRING 26.3.60.20001.altinityantalya) # end of autochange SET(VERSION_TWEAK 20001) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp index f4762c1a98002..f0f1981a52f7f 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp @@ -229,16 +229,14 @@ getClient(const S3::URI & url, const S3Settings & settings, ContextPtr context, } } -<<<<<<< HEAD - auto shared_cache = S3::ClientCacheRegistry::instance().getOrCreateCacheForKey(url.endpoint, url.bucket); -======= /// `gcp_oauth` adds an `Authorization` header to every request, so it must obey the same filter. if (boost::iequals(client_configuration.http_client, "gcp_oauth")) { HTTPHeaderEntries gcp_oauth_headers{{"Authorization", "Bearer"}}; context->getHTTPHeaderFilter().checkAndNormalizeHeaders(gcp_oauth_headers); } ->>>>>>> v26.3.60.2-lts + + auto shared_cache = S3::ClientCacheRegistry::instance().getOrCreateCacheForKey(url.endpoint, url.bucket); return S3::ClientFactory::instance().create( client_configuration, From 70b85f064ca0cc5fe6392b7de2c30ac5fcd5b508 Mon Sep 17 00:00:00 2001 From: blau-ai Date: Tue, 27 Oct 2026 10:36:55 +0000 Subject: [PATCH 46/46] Fix build: normalize headers after they are built in RestCatalog::sendRequest The conflict resolution left checkAndNormalizeHeaders(headers) before the 'headers' variable was declared, causing 'use of undeclared identifier headers'. Move the normalize call to right after headers is constructed, matching the pattern used in the GET request path. Co-Authored-By: Claude Opus 4.8 --- src/Databases/DataLake/RestCatalog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 08599e17b191b..43d85d2e61a23 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1108,7 +1108,6 @@ void RestCatalog::sendRequest(const String & endpoint, Poco::JSON::Object::Ptr r const std::string body_str = DB::removeEscapedSlashes(oss.str()); const auto & context = getContext(); - context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; if (!body_str.empty()) @@ -1127,6 +1126,7 @@ void RestCatalog::sendRequest(const String & endpoint, Poco::JSON::Object::Ptr r DB::HTTPHeaderEntries headers = getAuthHeaders(/* update_token = */ true, method, url, extra_headers, body_str); headers.emplace_back("Content-Type", "application/json"); + context->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(headers); auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withMethod(method)