pax_global_header00006660000000000000000000000064146745647500014534gustar00rootroot0000000000000052 comment=6d14111db40069c97a48d73bc02b8f6d4e315dba parse-document-model-python-0.2.0/000077500000000000000000000000001467456475000170765ustar00rootroot00000000000000parse-document-model-python-0.2.0/.devcontainer/000077500000000000000000000000001467456475000216355ustar00rootroot00000000000000parse-document-model-python-0.2.0/.devcontainer/devcontainer.json000066400000000000000000000013111467456475000252050ustar00rootroot00000000000000// For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/python { "name": "Python 3", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, // Use 'postCreateCommand' to run commands after the container is created. "postCreateCommand": "pip3 install --user -r requirements.txt -r requirements-dev.txt" // Configure tool-specific properties. // "customizations": {} } parse-document-model-python-0.2.0/.github/000077500000000000000000000000001467456475000204365ustar00rootroot00000000000000parse-document-model-python-0.2.0/.github/CONTRIBUTING.md000066400000000000000000000054731467456475000227000ustar00rootroot00000000000000# Contributing Contributions are **welcome** and will be fully **credited**. Please read and understand the contribution guide before creating an issue or pull request. ## Etiquette This project is open source, and as such, the maintainers give their free time to build and maintain the source code held within. They make the code freely available in the hope that it will be of use to other developers. It would be extremely unfair for them to suffer abuse or anger for their hard work. Please be considerate towards maintainers when raising issues or presenting pull requests. Let's show the world that developers are civilized and selfless people. It's the duty of the maintainer to ensure that all submissions to the project are of sufficient quality to benefit the project. Many developers have different skillsets, strengths, and weaknesses. Respect the maintainer's decision, and do not be upset or abusive if your submission is not used. ## Viability When requesting or submitting new features, first consider whether it might be useful to others. Open source projects are used by many developers, who may have entirely different needs to your own. Think about whether or not your feature is likely to be used by other users of the project. ## Procedure > [!NOTE] > Issue tracking is not currently enabled for this repository. We are organising it. Before filing an issue: - Attempt to replicate the problem, to ensure that it wasn't a coincidental incident. - Check to make sure your feature suggestion isn't already present within the project. - Check the pull requests tab to ensure that the bug doesn't have a fix in progress. - Check the pull requests tab to ensure that the feature isn't already in progress. Before submitting a pull request: - Check the codebase to ensure that your feature doesn't already exist. - Check the pull requests to ensure that another person hasn't already submitted the feature or fix. ## Requirements If the project maintainer has any additional requirements, you will find them listed here. - **Add tests!** - Your patch won't be accepted if it doesn't have tests. - **Document any change in behaviour** - Make sure the `README.md` and any other relevant documentation are kept up-to-date. - **Consider our release cycle** - We try to follow [SemVer v2.0.0](https://semver.org/). Randomly breaking public APIs is not an option. - **One pull request per feature** - If you want to do more than one thing, send multiple pull requests. - **Send coherent history** - Make sure each individual commit in your pull request is meaningful. If you had to make multiple intermediate commits while developing, please [squash them](https://www.git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Changing-Multiple-Commit-Messages) before submitting. **Happy coding**!parse-document-model-python-0.2.0/.github/SECURITY.md000066400000000000000000000002351467456475000222270ustar00rootroot00000000000000# Security Policy If you discover any security related issues, please email security@oneofftech.xyz instead of using the discussions or the issue tracker.parse-document-model-python-0.2.0/.github/workflows/000077500000000000000000000000001467456475000224735ustar00rootroot00000000000000parse-document-model-python-0.2.0/.github/workflows/ci.yml000066400000000000000000000016131467456475000236120ustar00rootroot00000000000000name: CI on: push: branches: - "main" pull_request: types: [opened, synchronize, reopened] permissions: contents: read jobs: test: name: Tests Python ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: fail-fast: true matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13-dev"] steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: | **/setup.py **/requirements*.txt - name: Install dependencies run: | pip install -r requirements.txt -r requirements-dev.txt - name: Run Testsuite run: | pytest parse-document-model-python-0.2.0/.github/workflows/release.yml000066400000000000000000000042051467456475000246370ustar00rootroot00000000000000name: Publish on PyPI on: push: branches: - "main" release: types: [created] permissions: contents: read jobs: test: name: Tests Python ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: fail-fast: true matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13-dev"] steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: | **/setup.py **/requirements*.txt - name: Install dependencies run: | pip install -r requirements.txt -r requirements-dev.txt - name: Run Testsuite run: | pytest build: runs-on: ubuntu-latest needs: test steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up Python 3.12 uses: actions/setup-python@v5 with: python-version: '3.12' cache: pip cache-dependency-path: | **/setup.py **/requirements*.txt - name: Install dependencies run: | pip install -r requirements.txt -r requirements-dev.txt - name: Build wheel run: | python setup.py sdist bdist_wheel twine check dist/* - name: Upload build for publishing uses: actions/upload-artifact@v4 with: name: parse_model_release if-no-files-found: error retention-days: 1 path: dist/* pypi-publish: name: Upload release to PyPI runs-on: ubuntu-latest needs: build environment: release permissions: id-token: write steps: - name: Download math result for job 1 uses: actions/download-artifact@v4 with: name: parse_model_release path: dist - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 if: github.event_name == 'release' with: print-hash: trueparse-document-model-python-0.2.0/.gitignore000066400000000000000000000016761467456475000211000ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg # PyCharm folder .idea/ # watchman # Watchman helper .watchmanconfig # OSX *.DS_Store # Atom plugin files and ctags .ftpconfig .ftpconfig.cson .ftpignore *.tags *.tags1 # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # dotenv .env # virtualenv .venv venv/ ENV/ # mypy .mypy_cache/ # vim *.swp parse-document-model-python-0.2.0/LICENSE000066400000000000000000000020731467456475000201050ustar00rootroot00000000000000MIT License Copyright (c) OneOffTech Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.parse-document-model-python-0.2.0/README.md000066400000000000000000000201061467456475000203540ustar00rootroot00000000000000![pypi](https://img.shields.io/pypi/v/parse-document-model.svg) [![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://docs.pydantic.dev/latest/contributing/#badges) [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) # Parse Document Model (Python) **Parse Document Model** (Python) provides Pydantic models for representing text documents using a hierarchical model. This library allows you to define documents as a hierarchy of (specialised) nodes where each node can represent a document, page, text, heading, body, and more. These models aim to preserve the underlying structure of text documents for further processing, such as creating a table of contents or transforming between formats, e.g. converting a parsed PDF to Markdown. - **Hierarchical structure**: The document is modelled as a hierarchy of nodes. Each node can represent a part of the document itself, pages, text. - **Rich text support**: Nodes can represent not only the content but also the formatting (e.g. bold, italic) applied to the text. - **Attributes**: Each node can have attributes that provide additional information such as page number, bounding box, etc. - **Built-in validation and types**: Built with [`Pydantic`](https://docs.pydantic.dev/latest/), ensuring type safety, validation and effortless creation of complex document structures. **Requirements** - Python 3.12 or above (Python 3.9, 3.10 and 3.11 are supported on best-effort). **Next steps** - [Explore the document model](#document-model-overview) - [Install the library and use the models](#getting-started) ## Document Model Overview We want to represent the document structure using a hierarchy so that the inherited structure is preserved when chapters, sections and headings are used. Consider a generic document with two pages, one heading per page and one paragraph of text. The resulting representation might be the following. ``` Document ├─Page │ ├─Text (category: heading) │ └─Text (category: body) └─Page ├─Text (category: heading) └─Text (category: body) ``` At a glance you can see the structure, the document is composed of two pages and there are two headings. To do so we defined a hierarchy around the concept of a Node, like a node in a graph. ### Node types ```mermaid classDiagram class Node Node <|-- StructuredNode Node <|-- Text StructuredNode <|-- Document StructuredNode <|-- Page ``` #### 1. **Node** (Base Class) This is the abstract class from which all other nodes inherit. Each node has: - `category`: The type of the node (e.g., `doc`, `page`, `heading`). - `attributes`: Optional field to attach extra data to a node. See [Attributes](#attributes). #### 2. **StructuredNode** This extends the [`Node`](#1-node-base-class). It is used to represent the hierarchy as a node whose content is a list of other nodes, such as like [`Document`](#3-document) and [`Page`](#4-page). - `content`: List of `Node`. #### 3. **Document** This is the root node of a document. - `category`: Always set to `"doc"`. - `attributes`: Document-wide attributes can be set here. - `content`: List of [`Page`](#4-page) nodes that form the document. #### 4. **Page** Represents a page in the document: - `category`: Always set to `"page"`. - `attributes`: Can contain metadata like page number. - `content`: List of [`Text`](#5-text) nodes on the page. #### 5. **Text** This node represent a paragraph, a heading or any text within the document. - `category`: The [category](#category) of the text within the document, e.g. `heading`, `title` - `content`: A string representing the textual content. - `marks`: List of [marks](#marks) applied to the text, such as bold, italic, etc. - `attributes`: Can contain metadata like the bounding box representing where this portion of text is located in the page. ### Category Each block of text is assigned a _category_. - `abstract`: The abstract of the document. - `acknowledgments`: Section acknowledging contributors. - `affiliation`: Author's institutional affiliation. - `appendix`: Text within an appendix. - `authors`: List of authors. - `body`: Main body text of the document. - `caption`: Caption associated with a figure or table. - `categories`: Categories or topics listed in the document. - `figure`: Represents a figure or an image. - `footer`: Represents the footer of the page. - `footnote`: Text at the bottom of the page providing additional information. - `formula`: Mathematical formula or equation. - `general-terms`: General terms section. - `heading`: Any heading within the document. - `keywords`: List of keywords. - `itemize-item`: Item in a list or bullet point. - `other`: Any other unclassified text. - `page-header`: Represents the header of the page. - `reference`: References or citations within the document. - `table`: Represents a table. - `title`: The title of the document. - `toc`: Table of contents. ### Marks Marks are used to add style or functionality to the text within a [`Text`](#5-text) node. For example, bold text, italic text, links and custom styles such as font or colour. **Mark Types** - `Bold`: Represents bold text. - `Italic`: Represents italic text. - `TextStyle`: Allows customization of font and color. - `Link`: Represents a hyperlink. Marks are validated and enforced with the help of `Pydantic` model validators. ### Attributes Attributes are optional fields that can store additional information for each node. Some predefined attributes are: - `DocumentAttributes`: General attributes for the document (currently reserved for the future). - `PageAttributes`: Specific page related attributes, such as the page number. - `TextAttributes`: Text related attributes, such as bounding boxes or level. - `BoundingBox`: A box that specifies the position of a text in the page. - `Level`: The specific level of the text within a document, for example, for headings. ## Getting started ### Installation Parse Document Model is distributed with PyPI. You can install it with `pip`. ```bash pip install parse-document-model ``` ### Quick Example Here’s how you can represent a simple document with one page and some text: ```python from document_model_python.document import Document, Page, Text doc = Document( category="doc", content=[ Page( category="page", content=[ Text( category="heading", content="Welcome to parse-document-model", marks=["bold"] ), Text( category="body", content="This is an example text using the document model." ) ] ) ] ) ``` ## Testing Parse Document Model is tested using [pytest](https://docs.pytest.org/en/stable/). Tests run for each commit and pull request. Install the dependencies. ```bash pip install -r requirements.txt -r requirements-dev.txt ``` Execute the test suite. ```bash pytest ``` ## Contributing Thank you for considering contributing to the Parse Document Model! The contribution guide can be found in the [CONTRIBUTING.md](./.github/CONTRIBUTING.md) file. > [NOTE] > Consider opening a [discussion](https://github.com/OneOffTech/parse-document-model-python/discussions) before submitting a pull request with changes to the model structures. ## Security Vulnerabilities Please review [our security policy](./.github/SECURITY.md) on how to report security vulnerabilities. ## Credits - [OneOffTech](https://github.com/OneOffTech) - [All Contributors](../../contributors) ## Supporters The project is provided and supported by [OneOff-Tech (UG)](https://oneofftech.de).

## Aknowledgements The format and structure takes inspiration from [ProseMirror](https://prosemirror.net/docs/ref/#model.Document_Schema). ## License The MIT License (MIT). Please see [License File](LICENSE.md) for more information. parse-document-model-python-0.2.0/parse_document_model/000077500000000000000000000000001467456475000232665ustar00rootroot00000000000000parse-document-model-python-0.2.0/parse_document_model/__init__.py000066400000000000000000000000451467456475000253760ustar00rootroot00000000000000from .document import Document, Page parse-document-model-python-0.2.0/parse_document_model/attributes.py000066400000000000000000000007311467456475000260270ustar00rootroot00000000000000from abc import ABC from typing import Optional from pydantic import BaseModel, Field class BoundingBox(BaseModel): min_x: float min_y: float max_x: float max_y: float page: int class Attributes(BaseModel, ABC): pass class DocumentAttributes(Attributes): pass class PageAttributes(Attributes): page: int class TextAttributes(Attributes): bounding_box: list[BoundingBox] = [] level: Optional[int] = Field(None, ge=1, le=4) parse-document-model-python-0.2.0/parse_document_model/document.py000066400000000000000000000056341467456475000254660ustar00rootroot00000000000000import warnings from abc import ABC from typing import List, Optional, Union from pydantic import BaseModel, Field, model_validator from parse_document_model.attributes import Attributes, PageAttributes, TextAttributes, DocumentAttributes from parse_document_model.marks import Mark, TextStyleMark, UrlMark class Node(BaseModel, ABC): """Base element of a document. A document is a hierarchy of nodes. Nodes could represent: document, pages, headings, etc. """ category: str = Field( ..., title="Node Type", description="The type of node. Examples are: `doc`, `page`, `heading`, `body`, etc. For an exhaustive list " "refers to the documentation.", ) attributes: Optional[Attributes] = Field( default=None, title="Node Attributes", description="Attributes related to the node. An example is the reference page." ) class StructuredNode(Node): content: List[Node] = Field( ..., title="Node Content", description="The content of the node. If it is a leaf node this is text, otherwise it could be a list of " "nodes.", ) class Text(Node): """The leaf node of a document. That's where the actual text is. """ attributes: Optional[TextAttributes] = TextAttributes() content: str = Field( ..., title="Content", description="The new field to hold the text content." ) marks: list[Union[Mark, TextStyleMark, UrlMark]] = [] text: Optional[str] = Field( None, title="Text", description="(Deprecated) This field is deprecated and will be removed in a future version. " "Use `content` instead." ) role: Optional[str] = Field( None, title="Node Type", description="(Deprecated) This field is deprecated and will be removed in a future version. " "Use `category` instead." ) @model_validator(mode="before") def handle_deprecations(self): if "text" in self and "content" not in self: warnings.warn("The use of `text` is deprecated and will be removed in a future version. " "Use `content` instead.", DeprecationWarning) self["content"] = self["text"] if "role" in self and "category" not in self: warnings.warn("The use of `role` is deprecated and will be removed in a future version. " "Use `category` instead.", DeprecationWarning) self["category"] = self["role"] return self class Page(StructuredNode): """The node that represents a document's page.""" category: str = "page" attributes: Optional[PageAttributes] = None content: list[Text] class Document(StructuredNode): """The root node of a document.""" category: str = "doc" attributes: Optional[DocumentAttributes] = None content: list[Page] parse-document-model-python-0.2.0/parse_document_model/marks.py000066400000000000000000000022231467456475000247540ustar00rootroot00000000000000from typing import Any from typing import Literal, Optional from pydantic import BaseModel, model_validator class Color(BaseModel): id: str r: int g: int b: int class Font(BaseModel): id: str name: str size: int class Mark(BaseModel): category: Literal['bold', 'italic', 'textStyle', 'link'] @model_validator(mode='before') def check_details(self: Any) -> Any: mark_type = self.get('category') if mark_type == 'textStyle': if 'color' not in self and 'font' not in self: raise ValueError('color or font must be provided when type is textStyle') if 'url' in self: raise ValueError('url should not be provided when type is textStyle') elif mark_type == 'link': if 'url' not in self: raise ValueError('url must be provided when type is link') if 'textStyle' in self: raise ValueError('textStyle should not be provided when type is link') return self class TextStyleMark(Mark): color: Optional[Color] = None font: Optional[Font] = None class UrlMark(Mark): url: str parse-document-model-python-0.2.0/requirements-dev.txt000066400000000000000000000000741467456475000231370ustar00rootroot00000000000000pytest~=8.3.3 setuptools~=75.0.0 twine~=5.1.1 wheel~=0.44.0 parse-document-model-python-0.2.0/requirements.txt000066400000000000000000000000201467456475000223520ustar00rootroot00000000000000pydantic~=2.9.1 parse-document-model-python-0.2.0/setup.py000066400000000000000000000020561467456475000206130ustar00rootroot00000000000000from codecs import open from os import path from setuptools import setup ROOT = path.abspath(path.dirname(__file__)) with open(path.join(ROOT, 'README.md'), encoding='utf-8') as f: long_description = f.read() setup( name='parse-document-model', version='0.2.0', description='Pydantic models for representing a text document as a hierarchical structure.', long_description=long_description, long_description_content_type='text/markdown', author='OneOffTech', author_email='info@oneofftech.xyz', license='MIT', url='https://github.com/OneOffTech/parse-document-model-python', project_urls={ 'Source': 'https://github.com/OneOffTech/parse-document-model-python', }, classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Operating System :: OS Independent' ], packages=['parse_document_model'], include_package_data=True, python_requires=">=3.9", install_requires=['pydantic>=2.9.0'] ) parse-document-model-python-0.2.0/test/000077500000000000000000000000001467456475000200555ustar00rootroot00000000000000parse-document-model-python-0.2.0/test/__init__.py000066400000000000000000000000001467456475000221540ustar00rootroot00000000000000parse-document-model-python-0.2.0/test/data/000077500000000000000000000000001467456475000207665ustar00rootroot00000000000000parse-document-model-python-0.2.0/test/data/extract-text-1.json000066400000000000000000000132431467456475000244560ustar00rootroot00000000000000{ "category": "doc", "content": [ { "category": "page", "attributes": { "page": 1 }, "content": [ { "role": "page-header", "text": "Type of document / Offer / Contract / Report", "marks": [ { "category": "textStyle", "color": { "r": 78, "b": 189, "g": 128, "id": "color-1" }, "font": { "name": "fira sans", "id": "font-300", "size": 18 } } ], "attributes": { "bounding_box": [ { "min_x": 62.1, "min_y": 565.0, "max_x": 427.2, "max_y": 577.6, "page": 1 } ] } }, { "role": "title", "text": "This is the title of the document, it", "marks": [ { "category": "textStyle", "color": { "r": 0, "b": 0, "g": 0, "id": "color-0" }, "font": { "name": "fira sans", "id": "font-300", "size": 30 } } ], "attributes": { "bounding_box": [ { "min_x": 62.1, "min_y": 532.1, "max_x": 514.6, "max_y": 554.7, "page": 1 } ] } }, { "role": "heading", "text": "can use multiple lines and grow a bit", "marks": [ { "category": "textStyle", "color": { "r": 0, "b": 0, "g": 0, "id": "color-0" }, "font": { "name": "fira sans", "id": "font-300", "size": 30 } } ], "attributes": { "bounding_box": [ { "min_x": 62.1, "min_y": 496.1, "max_x": 503.0, "max_y": 518.7, "page": 1 }, { "min_x": 62.1, "min_y": 460.1, "max_x": 98.7, "max_y": 482.7, "page": 1 } ] } }, { "role": "heading", "text": "Subtitle of the document", "marks": [ { "category": "textStyle", "color": { "r": 247, "b": 70, "g": 150, "id": "color-2" }, "font": { "name": "fira sans", "id": "font-300", "size": 22 } } ], "attributes": { "bounding_box": [ { "min_x": 62.1, "min_y": 431.6, "max_x": 296.5, "max_y": 447.1, "page": 1 } ] } }, { "role": "body", "text": "OneOff-Tech UG", "marks": [ { "category": "textStyle", "color": { "r": 0, "b": 0, "g": 0, "id": "color-0" }, "font": { "name": "fira sans-bold", "id": "font-301", "size": 11 } }, { "category": "bold" } ], "attributes": { "bounding_box": [ { "min_x": 62.1, "min_y": 209.0, "max_x": 253.5, "max_y": 217.5, "page": 1 } ] } } ] }, { "category": "page", "attributes": { "page": 2 }, "content": [ { "role": "heading", "text": "1 First chapter", "marks": [ { "category": "textStyle", "color": { "r": 53, "b": 145, "g": 96, "id": "color-4" }, "font": { "name": "fira sans", "id": "font-300", "size": 20 } } ], "attributes": { "bounding_box": [ { "min_x": 56.7, "min_y": 702.8, "max_x": 193.9, "max_y": 717.8, "page": 2 } ] } }, { "role": "body", "text": "This is an example text.", "marks": [ { "category": "textStyle", "color": { "r": 0, "b": 0, "g": 0, "id": "color-0" }, "font": { "name": "fira sans", "id": "font-300", "size": 11 } } ], "attributes": { "bounding_box": [ { "min_x": 56.7, "min_y": 665.0, "max_x": 504.2, "max_y": 687.3, "page": 2 } ] } } ] } ] }parse-document-model-python-0.2.0/test/data/extract-text-2.json000066400000000000000000000014451467456475000244600ustar00rootroot00000000000000{ "category": "doc", "content": [ { "category": "page", "attributes": { "page": 1 }, "content": [ { "role": "body", "text": "Type of document / Offer / Contract / Report This is the title of the document, it can use multiple lines and grow a bit Subtitle of the document OneOff-Tech", "marks": [], "attributes": { "bounding_box": [] } } ] }, { "category": "page", "attributes": { "page": 2 }, "content": [ { "role": "body", "text": "Section Heading 1 First chapter This is an example text.", "marks": [], "attributes": { "bounding_box": [] } } ] } ] }parse-document-model-python-0.2.0/test/data/extract-text-empty.json000066400000000000000000000002261467456475000254510ustar00rootroot00000000000000{ "category": "doc", "content": [ { "category": "page", "attributes": { "page": 1 }, "content": [] } ] }parse-document-model-python-0.2.0/test/test_validation.py000066400000000000000000000075371467456475000236340ustar00rootroot00000000000000import json import pytest from parse_document_model import Document, Page from parse_document_model.attributes import PageAttributes, TextAttributes from parse_document_model.marks import Mark, TextStyleMark, UrlMark def test_read_from_json(): filepaths = ["test/data/extract-text-1.json", "test/data/extract-text-2.json", "test/data/extract-text-empty.json"] for filepath in filepaths: doc_json = json.load(open(filepath, "r")) doc = Document(**doc_json) # Check the Document assert doc.category == "doc" assert isinstance(doc.content, list) # Check the Page for page in doc.content: assert isinstance(page, Page) assert page.category == "page" assert isinstance(page.attributes, PageAttributes) assert isinstance(page.content, list) # Check Text for text in page.content: assert text.category in ["page-header", "title", "heading", "body", "footer"] assert isinstance(text.content, str) assert isinstance(text.attributes, TextAttributes) assert isinstance(text.marks, list) # Check Marks for mark in text.marks: assert isinstance(mark, Mark) def test_style_marks(): text_style_mark_json = [{"category": "textStyle", "font": {"id": "1", "name": "test-font", "size": 1}}, {"category": "textStyle", "color": {"id": "1", "r": 0, "g": 0, "b": 0}}, {"category": "textStyle", "font": {"id": "1", "name": "test-font", "size": 1}, "color": {"id": "1", "r": 0, "g": 0, "b": 0}}, {"category": "textStyle"}, {"category": "textStyle", "url": "test-url"}] for mark_json in text_style_mark_json: if "font" in mark_json or "color" in mark_json: mark = TextStyleMark(**mark_json) assert isinstance(mark, TextStyleMark) else: with pytest.raises(ValueError): TextStyleMark(**mark_json) def test_url_marks(): url_mark_json = [{"category": "link", "url": "test-url"}, {"category": "link"}, {"category": "link", "font": {"id": "1", "name": "test-font", "size": 1}}, {"category": "link", "color": {"id": "1", "r": 0, "g": 0, "b": 0}}] for mark_json in url_mark_json: if "url" in mark_json: mark = UrlMark(**mark_json) assert isinstance(mark, UrlMark) else: with pytest.raises(ValueError): UrlMark(**mark_json) def test_text_attributes_level(): valid_text_attributes = [ {"bounding_box": [], "level": 1}, {"bounding_box": [], "level": 2}, {"bounding_box": [], "level": 3}, {"bounding_box": [], "level": 4}, {"bounding_box": [], "level": None}, {"bounding_box": []}, {} ] for attributes_json in valid_text_attributes: text_attributes = TextAttributes(**attributes_json) assert isinstance(text_attributes, TextAttributes) assert isinstance(text_attributes.level, (int, type(None))) if text_attributes.level is not None: assert text_attributes.level in range(1, 5) assert attributes_json["level"] == text_attributes.level else: assert "level" not in attributes_json or attributes_json["level"] is None invalid_text_attributes = [ {"bounding_box": [], "level": -1}, {"bounding_box": [], "level": "invalid"}, {"bounding_box": [], "level": 2.5}, {"bounding_box": [], "level": 5}, ] for attributes_json in invalid_text_attributes: with pytest.raises(ValueError): TextAttributes(**attributes_json)