Skip to content

edspdf.extractors

functional

get_blocs(layout)

Extract text blocs from a PDFMiner layout generator.

Arguments

layout: PDFMiner layout generator.

YIELDS DESCRIPTION
bloc

Text bloc

TYPE: Iterator[Tuple[LTTextBoxHorizontal, int, float, float]]

Source code in edspdf/extractors/functional.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
def get_blocs(
    layout: Iterator[LTPage],
) -> Iterator[Tuple[LTTextBoxHorizontal, int, float, float]]:
    """
    Extract text blocs from a PDFMiner layout generator.

    Arguments
    ---------
    layout:
        PDFMiner layout generator.

    Yields
    ------
    bloc :
        Text bloc
    """

    for i, page in enumerate(layout):

        width = page.width
        height = page.height

        for bloc in page:
            if isinstance(bloc, LTTextBoxHorizontal):
                yield bloc, i, width, height

get_lines(layout)

Extract lines from a PDFMiner layout object.

The line is reframed such that the origin is the top left corner.

PARAMETER DESCRIPTION
layout

PDFMiner layout object.

TYPE: Iterator[LTPage]

YIELDS DESCRIPTION
Iterator[Line]

Single line object.
Source code in edspdf/extractors/functional.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def get_lines(layout: Iterator[LTPage]) -> Iterator[Line]:
    """
    Extract lines from a PDFMiner layout object.

    The line is reframed such that the origin is the top left corner.

    Parameters
    ----------
    layout : Iterator[LTPage]
        PDFMiner layout object.

    Yields
    -------
    Iterator[Line]
        Single line object.
    """
    for b, (bloc, p, w, h) in enumerate(get_blocs(layout)):
        for line in bloc:
            text, styles = extract_style(line, width=w, height=h)
            yield Line(
                page=p,
                bloc=b,
                x0=line.x0 / w,
                x1=line.x1 / w,
                y0=1 - line.y1 / h,
                y1=1 - line.y0 / h,
                page_width=w,
                page_height=h,
                text=text,
                styles=styles,
            )

remove_outside_lines(lines, strict_mode=False)

Filter out lines that are outside the canvas.

PARAMETER DESCRIPTION
lines

Dataframe of extracted lines

TYPE: pd.DataFrame

strict_mode

Whether to remove the line if any part of it is outside the canvas, by default False

TYPE: bool, optional DEFAULT: False

RETURNS DESCRIPTION
pd.DataFrame

Filtered lines.
Source code in edspdf/extractors/functional.py 
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def remove_outside_lines(
    lines: pd.DataFrame,
    strict_mode: bool = False,
) -> pd.DataFrame:
    """
    Filter out lines that are outside the canvas.

    Parameters
    ----------
    lines : pd.DataFrame
        Dataframe of extracted lines
    strict_mode : bool, optional
        Whether to remove the line if any part of it is outside the canvas,
        by default False

    Returns
    -------
    pd.DataFrame
        Filtered lines.
    """
    if strict_mode:
        lower = lines[["x0", "y0"]].min(axis=1) >= 0
        upper = lines[["x1", "y1"]].max(axis=1) <= 1
        lines = lines[lower & upper]
    else:
        below = lines[["x1", "y1"]].max(axis=1) < 0
        above = lines[["x0", "y0"]].min(axis=1) > 0
        lines = lines[~(below | above)]
    return lines

base

BaseExtractor

Bases: ABC

Source code in edspdf/extractors/base.py 
 6
 7
 8
 9
10
11
12
13
14
class BaseExtractor(ABC):
    @abstractmethod
    def extract(self, pdf: bytes) -> pd.DataFrame:
        """
        Handles the extraction
        """

    def __call__(self, pdf: bytes) -> pd.DataFrame:
        return self.extract(pdf)

extract(pdf) abstractmethod

Handles the extraction

Source code in edspdf/extractors/base.py 
 7
 8
 9
10
11
@abstractmethod
def extract(self, pdf: bytes) -> pd.DataFrame:
    """
    Handles the extraction
    """

pdfminer

PdfMinerExtractor

Bases: BaseExtractor

Extractor object. Given a PDF byte stream, produces a list of blocs.

PARAMETER DESCRIPTION
line_overlap

See PDFMiner documentation

TYPE: float DEFAULT: 0.5

char_margin

See PDFMiner documentation

TYPE: float DEFAULT: 2.0

line_margin

See PDFMiner documentation

TYPE: float DEFAULT: 0.5

word_margin

See PDFMiner documentation

TYPE: float DEFAULT: 0.1

boxes_flow

See PDFMiner documentation

TYPE: Optional[float] DEFAULT: 0.5

detect_vertical

See PDFMiner documentation

TYPE: bool DEFAULT: False

all_texts

See PDFMiner documentation

TYPE: bool DEFAULT: False

Source code in edspdf/extractors/pdfminer.py 
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
@registry.extractors.register("pdfminer.v1")
class PdfMinerExtractor(BaseExtractor):
    """
    Extractor object. Given a PDF byte stream, produces a list of blocs.

    Parameters
    ----------
    line_overlap : float
        See PDFMiner documentation
    char_margin : float
        See PDFMiner documentation
    line_margin : float
        See PDFMiner documentation
    word_margin : float
        See PDFMiner documentation
    boxes_flow : Optional[float]
        See PDFMiner documentation
    detect_vertical : bool
        See PDFMiner documentation
    all_texts : bool
        See PDFMiner documentation
    """

    def __init__(
        self,
        line_overlap: float = 0.5,
        char_margin: float = 2.0,
        line_margin: float = 0.5,
        word_margin: float = 0.1,
        boxes_flow: Optional[float] = 0.5,
        detect_vertical: bool = False,
        all_texts: bool = False,
    ):

        self.laparams = LAParams(
            line_overlap=line_overlap,
            char_margin=char_margin,
            line_margin=line_margin,
            word_margin=word_margin,
            boxes_flow=boxes_flow,
            detect_vertical=detect_vertical,
            all_texts=all_texts,
        )

    def generate_lines(self, pdf: bytes) -> pd.DataFrame:
        """
        Generates dataframe from all blocs in the PDF.

        Arguments
        ---------
        pdf:
            Byte stream representing the PDF.

        Returns
        -------
        pd.DataFrame :
            DataFrame representing the blocs.
        """

        pdf_stream = BytesIO(pdf)

        layout = extract_pages(pdf_stream, laparams=self.laparams)
        lines = list(get_lines(layout))

        if not lines:
            return pd.DataFrame(
                columns=[
                    "page",
                    "bloc",
                    "x0",
                    "x1",
                    "y0",
                    "y1",
                    "page_width",
                    "page_height",
                    "text",
                    "styles",
                ]
            )

        df = pd.DataFrame.from_records([line.dict() for line in lines])
        df["line_id"] = range(len(df))

        return df

    def extract(self, pdf: bytes) -> pd.DataFrame:
        """
        Process a single PDF document.

        Parameters
        ----------
        pdf : bytes
            Raw byte representation of the PDF document.

        Returns
        -------
        pd.DataFrame
            DataFrame containing one row for each line extracted using PDFMiner.
        """

        lines = self.generate_lines(pdf)

        # Remove empty lines
        lines = lines[lines.text.str.len() > 0]

        # Remove lines that are outside the page
        lines = remove_outside_lines(lines, strict_mode=True)

        return lines

generate_lines(pdf)

Generates dataframe from all blocs in the PDF.

Arguments

pdf: Byte stream representing the PDF.

RETURNS DESCRIPTION
pd.DataFrame

DataFrame representing the blocs.
Source code in edspdf/extractors/pdfminer.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def generate_lines(self, pdf: bytes) -> pd.DataFrame:
    """
    Generates dataframe from all blocs in the PDF.

    Arguments
    ---------
    pdf:
        Byte stream representing the PDF.

    Returns
    -------
    pd.DataFrame :
        DataFrame representing the blocs.
    """

    pdf_stream = BytesIO(pdf)

    layout = extract_pages(pdf_stream, laparams=self.laparams)
    lines = list(get_lines(layout))

    if not lines:
        return pd.DataFrame(
            columns=[
                "page",
                "bloc",
                "x0",
                "x1",
                "y0",
                "y1",
                "page_width",
                "page_height",
                "text",
                "styles",
            ]
        )

    df = pd.DataFrame.from_records([line.dict() for line in lines])
    df["line_id"] = range(len(df))

    return df

extract(pdf)

Process a single PDF document.

PARAMETER DESCRIPTION
pdf

Raw byte representation of the PDF document.

TYPE: bytes

RETURNS DESCRIPTION
pd.DataFrame

DataFrame containing one row for each line extracted using PDFMiner.
Source code in edspdf/extractors/pdfminer.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
def extract(self, pdf: bytes) -> pd.DataFrame:
    """
    Process a single PDF document.

    Parameters
    ----------
    pdf : bytes
        Raw byte representation of the PDF document.

    Returns
    -------
    pd.DataFrame
        DataFrame containing one row for each line extracted using PDFMiner.
    """

    lines = self.generate_lines(pdf)

    # Remove empty lines
    lines = lines[lines.text.str.len() > 0]

    # Remove lines that are outside the page
    lines = remove_outside_lines(lines, strict_mode=True)

    return lines

style

models

BaseStyle

Bases: BaseModel

Model acting as an abstraction for a style.

Source code in edspdf/extractors/style/models.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
class BaseStyle(BaseModel):
    """
    Model acting as an abstraction for a style.
    """

    fontname: Optional[str] = None

    font: str
    style: str
    size: float
    upright: bool

    x0: float
    x1: float
    y0: float
    y1: float

Style

Bases: BaseStyle

Model acting as an abstraction for a style.

Source code in edspdf/extractors/style/models.py 
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
class Style(BaseStyle):
    """
    Model acting as an abstraction for a style.
    """

    @classmethod
    def from_fontname(
        cls,
        fontname: str,
        size: float,
        upright: bool,
        x0: float,
        x1: float,
        y0: float,
        y1: float,
    ) -> "Style":
        """
        Constructor using the compound `fontname` representation.

        Parameters
        ----------
        fontname : str
            Compound description of the font. Often `Arial`,
            `Arial,Bold` or `Arial-Bold`
        size : float
            Character size.
        upright : bool
            Whether the character is upright.

        Returns
        -------
        Style
            Style representation.
        """
        # Round the size to avoid floating point aberrations.
        size = round(size, 2)

        s = SEP_PATTERN.split(fontname)

        font = s.pop(0)

        if s:
            style = s[-1]
        else:
            style = "Normal"

        s = Style(
            fontname=fontname,
            font=font,
            style=style,
            size=size,
            upright=upright,
            x0=x0,
            x1=x1,
            y0=y0,
            y1=y1,
        )

        return s

    @classmethod
    def from_char(
        cls,
        char: LTChar,
        width: float,
        height: float,
    ):
        return cls.from_fontname(
            fontname=char.fontname,
            size=char.size,
            upright=char.upright,
            x0=char.x0 / width,
            x1=char.x1 / width,
            y0=1 - char.y1 / height,
            y1=1 - char.y0 / height,
        )

    def __eq__(self, other: "Style") -> bool:
        """
        Computes equality between two styles.

        Parameters
        ----------
        other : Style
            Style object to compare.

        Returns
        -------
        bool
            Whether the two styles are equal.
        """

        s = (self.font, self.style, round(self.size, 2), self.upright)
        o = (other.font, other.style, round(other.size, 2), other.upright)

        return s == o

    def __add__(self, other: "Style") -> "Style":

        if self != other:
            raise ValueError("You cannot add two different styles")

        st = self.copy()

        st.x0 = min(self.x0, other.x0)
        st.x1 = max(self.x1, other.x1)
        st.y0 = min(self.y0, other.y0)
        st.y1 = max(self.y1, other.y1)

        return st
from_fontname(fontname, size, upright, x0, x1, y0, y1) classmethod

Constructor using the compound fontname representation.

PARAMETER DESCRIPTION
fontname

Compound description of the font. Often Arial, Arial,Bold or Arial-Bold

TYPE: str

size

Character size.

TYPE: float

upright

Whether the character is upright.

TYPE: bool

RETURNS DESCRIPTION
Style

Style representation.
Source code in edspdf/extractors/style/models.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
@classmethod
def from_fontname(
    cls,
    fontname: str,
    size: float,
    upright: bool,
    x0: float,
    x1: float,
    y0: float,
    y1: float,
) -> "Style":
    """
    Constructor using the compound `fontname` representation.

    Parameters
    ----------
    fontname : str
        Compound description of the font. Often `Arial`,
        `Arial,Bold` or `Arial-Bold`
    size : float
        Character size.
    upright : bool
        Whether the character is upright.

    Returns
    -------
    Style
        Style representation.
    """
    # Round the size to avoid floating point aberrations.
    size = round(size, 2)

    s = SEP_PATTERN.split(fontname)

    font = s.pop(0)

    if s:
        style = s[-1]
    else:
        style = "Normal"

    s = Style(
        fontname=fontname,
        font=font,
        style=style,
        size=size,
        upright=upright,
        x0=x0,
        x1=x1,
        y0=y0,
        y1=y1,
    )

    return s
__eq__(other)

Computes equality between two styles.

PARAMETER DESCRIPTION
other

Style object to compare.

TYPE: Style

RETURNS DESCRIPTION
bool

Whether the two styles are equal.
Source code in edspdf/extractors/style/models.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def __eq__(self, other: "Style") -> bool:
    """
    Computes equality between two styles.

    Parameters
    ----------
    other : Style
        Style object to compare.

    Returns
    -------
    bool
        Whether the two styles are equal.
    """

    s = (self.font, self.style, round(self.size, 2), self.upright)
    o = (other.font, other.style, round(other.size, 2), other.upright)

    return s == o

StyledText

Bases: BaseModel

Abstraction of a word, containing the style and the text.

Source code in edspdf/extractors/style/models.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
class StyledText(BaseModel):
    """
    Abstraction of a word, containing the style and the text.
    """

    text: str
    style: Style

    @classmethod
    def from_char(
        cls,
        char: LTChar,
        width: float,
        height: float,
    ):
        return StyledText(
            text=SPACE_PATTERN.sub(" ", char._text),
            style=Style.from_char(char, width=width, height=height),
        )

    def add_space(self) -> None:
        self.text = f"{self.text.rstrip()} "

    def rstrip(self) -> None:
        self.text = self.text.rstrip()

    def __add__(self, other: "StyledText") -> "StyledText":

        st = StyledText(
            text=self.text + other.text,
            style=self.style + other.style,
        )

        return st

    def __iadd__(self, other: "StyledText") -> "StyledText":
        return self + other