import os, struct
import pyarrow as pa
import pyarrow.parquet as pq

TARGET = 120 * 1024 * 1024  # 120 MiB

table = pa.table({"x": pa.array([1, 2, 3], type=pa.int32())})

payload = b"A" * TARGET
meta = dict(table.schema.metadata or {})
meta[b"giant_meta"] = payload
table2 = table.replace_schema_metadata(meta)

out = "too_big_footer_kv.parquet"
pq.write_table(table2, out, compression="snappy")

# Read parquet footer length (stored in last 8 bytes: <footer_len><PAR1>)
with open(out, "rb") as f:
    f.seek(-8, os.SEEK_END)
    footer_len = struct.unpack("<I", f.read(4))[0]
    magic = f.read(4)

print("file =", out)
print("footer_len =", footer_len, "bytes")
print("magic =", magic)
print("file_size =", os.path.getsize(out), "bytes")

arthur :) select * from file('too_big_footer_kv.parquet') SETTINGS input_format_parquet_use_native_reader_v3=1;

SELECT *
FROM file('too_big_footer_kv.parquet')
SETTINGS input_format_parquet_use_native_reader_v3 = 1

Query id: 4f316fce-9807-4702-a50b-fc7ef930034b


Elapsed: 0.683 sec. 

Received exception from server (version 25.8.9):
Code: 636. DB::Exception: Received from localhost:9000. DB::Exception: The table structure cannot be extracted from a Parquet format file. Error:
Code: 1001. DB::Exception: apache::thrift::transport::TTransportException: MaxMessageSize reached. (STD_EXCEPTION) (version 25.8.9.20000.altinityantalya).
You can specify the structure manually: (in file/uri /home/laptop/work/altinity/export_replicated_mt_partition/programs/server/user_files/too_big_footer_kv.parquet). (CANNOT_EXTRACT_TABLE_STRUCTURE)

arthur :) select * from file('too_big_footer_kv.parquet') SETTINGS input_format_parquet_use_native_reader_v3=1;

SELECT *
FROM file('too_big_footer_kv.parquet')
SETTINGS input_format_parquet_use_native_reader_v3 = 1

Query id: 386acd35-c2c0-4419-996c-a35a969f4468

   ┌─x─┐
1. │ 1 │
2. │ 2 │
3. │ 3 │
   └───┘

3 rows in set. Elapsed: 4.553 sec.