Skip to content

Commit 723809f

Browse files
bbc2bbc2
committed
Print a warning on malformed line
If a line can't be parsed (e.g. `FOO: BAR` is invalid), it is printed as a warning on stderr. As usual, after such an error, Python-dotenv tries to parse the rest of the file and the application can use the successfully parsed variable bindings.
1 parent b1e83ca commit 723809f

3 files changed

Lines changed: 153 additions & 68 deletions

File tree

‎src/dotenv/main.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from __future__ import absolute_import, print_function, unicode_literals
33

44
import io
5+
import logging
56
import os
67
import re
78
import shutil
@@ -11,8 +12,10 @@
1112
from collections import OrderedDict
1213
from contextlib import contextmanager
1314

14-
from .compat import StringIO, PY2, to_env, IS_TYPE_CHECKING
15-
from .parser import parse_stream
15+
from .compat import IS_TYPE_CHECKING, PY2, StringIO, to_env
16+
from .parser import Binding, parse_stream
17+
18+
logger = logging.getLogger(__name__)
1619

1720
if IS_TYPE_CHECKING:
1821
from typing import (
@@ -31,6 +34,17 @@
3134
__posix_variable = re.compile(r'\$\{[^\}]*\}') # type: Pattern[Text]
3235

3336

37+
def with_warn_for_invalid_lines(mappings):
38+
# type: (Iterator[Binding]) -> Iterator[Binding]
39+
for mapping in mappings:
40+
if mapping.key is None or mapping.value is None:
41+
logger.warning(
42+
"Python-dotenv could not parse statement starting at line %s",
43+
mapping.original.line,
44+
)
45+
yield mapping
46+
47+
3448
class DotEnv():
3549

3650
def __init__(self, dotenv_path, verbose=False, encoding=None):
@@ -66,7 +80,7 @@ def dict(self):
6680
def parse(self):
6781
# type: () -> Iterator[Tuple[Text, Text]]
6882
with self._get_stream() as stream:
69-
for mapping in parse_stream(stream):
83+
for mapping in with_warn_for_invalid_lines(parse_stream(stream)):
7084
if mapping.key is not None and mapping.value is not None:
7185
yield mapping.key, mapping.value
7286

@@ -143,12 +157,12 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"):
143157

144158
with rewrite(dotenv_path) as (source, dest):
145159
replaced = False
146-
for mapping in parse_stream(source):
160+
for mapping in with_warn_for_invalid_lines(parse_stream(source)):
147161
if mapping.key == key_to_set:
148162
dest.write(line_out)
149163
replaced = True
150164
else:
151-
dest.write(mapping.original)
165+
dest.write(mapping.original.string)
152166
if not replaced:
153167
dest.write(line_out)
154168

@@ -169,11 +183,11 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"):
169183

170184
removed = False
171185
with rewrite(dotenv_path) as (source, dest):
172-
for mapping in parse_stream(source):
186+
for mapping in with_warn_for_invalid_lines(parse_stream(source)):
173187
if mapping.key == key_to_unset:
174188
removed = True
175189
else:
176-
dest.write(mapping.original)
190+
dest.write(mapping.original.string)
177191

178192
if not removed:
179193
warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) # type: ignore

‎src/dotenv/parser.py‎

Lines changed: 80 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
import codecs
22
import re
33

4-
from .compat import to_text, IS_TYPE_CHECKING
5-
4+
from .compat import IS_TYPE_CHECKING, to_text
65

76
if IS_TYPE_CHECKING:
87
from typing import ( # noqa:F401
@@ -16,6 +15,7 @@ def make_regex(string, extra_flags=0):
1615
return re.compile(to_text(string), re.UNICODE | extra_flags)
1716

1817

18+
_newline = make_regex(r"(\r\n|\n|\r)")
1919
_whitespace = make_regex(r"\s*", extra_flags=re.MULTILINE)
2020
_export = make_regex(r"(?:export[^\S\r\n]+)?")
2121
_single_quoted_key = make_regex(r"'([^']+)'")
@@ -36,14 +36,62 @@ def make_regex(string, extra_flags=0):
3636
# when we are type checking, and the linter is upset if we
3737
# re-import
3838
import typing
39-
Binding = typing.NamedTuple("Binding", [("key", typing.Optional[typing.Text]),
40-
("value", typing.Optional[typing.Text]),
41-
("original", typing.Text)])
39+
40+
Original = typing.NamedTuple(
41+
"Original",
42+
[
43+
("string", typing.Text),
44+
("line", int),
45+
],
46+
)
47+
48+
Binding = typing.NamedTuple(
49+
"Binding",
50+
[
51+
("key", typing.Optional[typing.Text]),
52+
("value", typing.Optional[typing.Text]),
53+
("original", Original),
54+
],
55+
)
4256
except ImportError:
4357
from collections import namedtuple
44-
Binding = namedtuple("Binding", ["key", # type: ignore
45-
"value",
46-
"original"]) # type: Tuple[Optional[Text], Optional[Text], Text]
58+
Original = namedtuple( # type: ignore
59+
"Original",
60+
[
61+
"string",
62+
"line",
63+
],
64+
)
65+
Binding = namedtuple( # type: ignore
66+
"Binding",
67+
[
68+
"key",
69+
"value",
70+
"original",
71+
],
72+
)
73+
74+
75+
class Position:
76+
def __init__(self, chars, line):
77+
# type: (int, int) -> None
78+
self.chars = chars
79+
self.line = line
80+
81+
@classmethod
82+
def start(cls):
83+
# type: () -> Position
84+
return cls(chars=0, line=1)
85+
86+
def set(self, other):
87+
# type: (Position) -> None
88+
self.chars = other.chars
89+
self.line = other.line
90+
91+
def advance(self, string):
92+
# type: (Text) -> None
93+
self.chars += len(string)
94+
self.line += len(re.findall(_newline, string))
4795

4896

4997
class Error(Exception):
@@ -54,39 +102,42 @@ class Reader:
54102
def __init__(self, stream):
55103
# type: (IO[Text]) -> None
56104
self.string = stream.read()
57-
self.position = 0
58-
self.mark = 0
105+
self.position = Position.start()
106+
self.mark = Position.start()
59107

60108
def has_next(self):
61109
# type: () -> bool
62-
return self.position < len(self.string)
110+
return self.position.chars < len(self.string)
63111

64112
def set_mark(self):
65113
# type: () -> None
66-
self.mark = self.position
114+
self.mark.set(self.position)
67115

68116
def get_marked(self):
69-
# type: () -> Text
70-
return self.string[self.mark:self.position]
117+
# type: () -> Original
118+
return Original(
119+
string=self.string[self.mark.chars:self.position.chars],
120+
line=self.mark.line,
121+
)
71122

72123
def peek(self, count):
73124
# type: (int) -> Text
74-
return self.string[self.position:self.position + count]
125+
return self.string[self.position.chars:self.position.chars + count]
75126

76127
def read(self, count):
77128
# type: (int) -> Text
78-
result = self.string[self.position:self.position + count]
129+
result = self.string[self.position.chars:self.position.chars + count]
79130
if len(result) < count:
80131
raise Error("read: End of string")
81-
self.position += count
132+
self.position.advance(result)
82133
return result
83134

84135
def read_regex(self, regex):
85136
# type: (Pattern[Text]) -> Sequence[Text]
86-
match = regex.match(self.string, self.position)
137+
match = regex.match(self.string, self.position.chars)
87138
if match is None:
88139
raise Error("read_regex: Pattern not found")
89-
self.position = match.end()
140+
self.position.advance(self.string[match.start():match.end()])
90141
return match.groups()
91142

92143

@@ -147,10 +198,18 @@ def parse_binding(reader):
147198
value = parse_value(reader)
148199
reader.read_regex(_comment)
149200
reader.read_regex(_end_of_line)
150-
return Binding(key=key, value=value, original=reader.get_marked())
201+
return Binding(
202+
key=key,
203+
value=value,
204+
original=reader.get_marked(),
205+
)
151206
except Error:
152207
reader.read_regex(_rest_of_line)
153-
return Binding(key=None, value=None, original=reader.get_marked())
208+
return Binding(
209+
key=None,
210+
value=None,
211+
original=reader.get_marked(),
212+
)
154213

155214

156215
def parse_stream(stream):

‎tests/test_parser.py‎

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,83 +2,95 @@
22
import pytest
33

44
from dotenv.compat import StringIO
5-
from dotenv.parser import Binding, parse_stream
5+
from dotenv.parser import Binding, Original, parse_stream
66

77

88
@pytest.mark.parametrize("test_input,expected", [
99
(u"", []),
10-
(u"a=b", [Binding(key=u"a", value=u"b", original=u"a=b")]),
11-
(u"'a'=b", [Binding(key=u"a", value=u"b", original=u"'a'=b")]),
12-
(u"[=b", [Binding(key=u"[", value=u"b", original=u"[=b")]),
13-
(u" a = b ", [Binding(key=u"a", value=u"b", original=u" a = b ")]),
14-
(u"export a=b", [Binding(key=u"a", value=u"b", original=u"export a=b")]),
15-
(u" export 'a'=b", [Binding(key=u"a", value=u"b", original=u" export 'a'=b")]),
16-
(u"# a=b", [Binding(key=None, value=None, original=u"# a=b")]),
17-
(u"a=b#c", [Binding(key=u"a", value=u"b#c", original=u"a=b#c")]),
18-
(u'a=b # comment', [Binding(key=u"a", value=u"b", original=u"a=b # comment")]),
19-
(u"a=b space ", [Binding(key=u"a", value=u"b space", original=u"a=b space ")]),
20-
(u"a='b space '", [Binding(key=u"a", value=u"b space ", original=u"a='b space '")]),
21-
(u'a="b space "', [Binding(key=u"a", value=u"b space ", original=u'a="b space "')]),
22-
(u"export export_a=1", [Binding(key=u"export_a", value=u"1", original=u"export export_a=1")]),
23-
(u"export port=8000", [Binding(key=u"port", value=u"8000", original=u"export port=8000")]),
24-
(u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=u'a="b\nc"')]),
25-
(u"a='b\nc'", [Binding(key=u"a", value=u"b\nc", original=u"a='b\nc'")]),
26-
(u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=u'a="b\nc"')]),
27-
(u'a="b\\nc"', [Binding(key=u"a", value=u'b\nc', original=u'a="b\\nc"')]),
28-
(u"a='b\\nc'", [Binding(key=u"a", value=u'b\\nc', original=u"a='b\\nc'")]),
29-
(u'a="b\\"c"', [Binding(key=u"a", value=u'b"c', original=u'a="b\\"c"')]),
30-
(u"a='b\\'c'", [Binding(key=u"a", value=u"b'c", original=u"a='b\\'c'")]),
31-
(u"a=à", [Binding(key=u"a", value=u"à", original=u"a=à")]),
32-
(u'a="à"', [Binding(key=u"a", value=u"à", original=u'a="à"')]),
33-
(u'garbage', [Binding(key=None, value=None, original=u"garbage")]),
10+
(u"a=b", [Binding(key=u"a", value=u"b", original=Original(string=u"a=b", line=1))]),
11+
(u"'a'=b", [Binding(key=u"a", value=u"b", original=Original(string=u"'a'=b", line=1))]),
12+
(u"[=b", [Binding(key=u"[", value=u"b", original=Original(string=u"[=b", line=1))]),
13+
(u" a = b ", [Binding(key=u"a", value=u"b", original=Original(string=u" a = b ", line=1))]),
14+
(u"export a=b", [Binding(key=u"a", value=u"b", original=Original(string=u"export a=b", line=1))]),
15+
(u" export 'a'=b", [Binding(key=u"a", value=u"b", original=Original(string=u" export 'a'=b", line=1))]),
16+
(u"# a=b", [Binding(key=None, value=None, original=Original(string=u"# a=b", line=1))]),
17+
(u"a=b#c", [Binding(key=u"a", value=u"b#c", original=Original(string=u"a=b#c", line=1))]),
18+
(u'a=b # comment', [Binding(key=u"a", value=u"b", original=Original(string=u"a=b # comment", line=1))]),
19+
(u"a=b space ", [Binding(key=u"a", value=u"b space", original=Original(string=u"a=b space ", line=1))]),
20+
(u"a='b space '", [Binding(key=u"a", value=u"b space ", original=Original(string=u"a='b space '", line=1))]),
21+
(u'a="b space "', [Binding(key=u"a", value=u"b space ", original=Original(string=u'a="b space "', line=1))]),
22+
(
23+
u"export export_a=1",
24+
[
25+
Binding(key=u"export_a", value=u"1", original=Original(string=u"export export_a=1", line=1))
26+
],
27+
),
28+
(u"export port=8000", [Binding(key=u"port", value=u"8000", original=Original(string=u"export port=8000", line=1))]),
29+
(u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=Original(string=u'a="b\nc"', line=1))]),
30+
(u"a='b\nc'", [Binding(key=u"a", value=u"b\nc", original=Original(string=u"a='b\nc'", line=1))]),
31+
(u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=Original(string=u'a="b\nc"', line=1))]),
32+
(u'a="b\\nc"', [Binding(key=u"a", value=u'b\nc', original=Original(string=u'a="b\\nc"', line=1))]),
33+
(u"a='b\\nc'", [Binding(key=u"a", value=u'b\\nc', original=Original(string=u"a='b\\nc'", line=1))]),
34+
(u'a="b\\"c"', [Binding(key=u"a", value=u'b"c', original=Original(string=u'a="b\\"c"', line=1))]),
35+
(u"a='b\\'c'", [Binding(key=u"a", value=u"b'c", original=Original(string=u"a='b\\'c'", line=1))]),
36+
(u"a=à", [Binding(key=u"a", value=u"à", original=Original(string=u"a=à", line=1))]),
37+
(u'a="à"', [Binding(key=u"a", value=u"à", original=Original(string=u'a="à"', line=1))]),
38+
(u'garbage', [Binding(key=None, value=None, original=Original(string=u"garbage", line=1))]),
3439
(
3540
u"a=b\nc=d",
3641
[
37-
Binding(key=u"a", value=u"b", original=u"a=b\n"),
38-
Binding(key=u"c", value=u"d", original=u"c=d"),
42+
Binding(key=u"a", value=u"b", original=Original(string=u"a=b\n", line=1)),
43+
Binding(key=u"c", value=u"d", original=Original(string=u"c=d", line=2)),
44+
],
45+
),
46+
(
47+
u"a=b\rc=d",
48+
[
49+
Binding(key=u"a", value=u"b", original=Original(string=u"a=b\r", line=1)),
50+
Binding(key=u"c", value=u"d", original=Original(string=u"c=d", line=2)),
3951
],
4052
),
4153
(
4254
u"a=b\r\nc=d",
4355
[
44-
Binding(key=u"a", value=u"b", original=u"a=b\r\n"),
45-
Binding(key=u"c", value=u"d", original=u"c=d"),
56+
Binding(key=u"a", value=u"b", original=Original(string=u"a=b\r\n", line=1)),
57+
Binding(key=u"c", value=u"d", original=Original(string=u"c=d", line=2)),
4658
],
4759
),
4860
(
4961
u'a=\nb=c',
5062
[
51-
Binding(key=u"a", value=u'', original=u'a=\n'),
52-
Binding(key=u"b", value=u'c', original=u"b=c"),
63+
Binding(key=u"a", value=u'', original=Original(string=u'a=\n', line=1)),
64+
Binding(key=u"b", value=u'c', original=Original(string=u"b=c", line=2)),
5365
]
5466
),
5567
(
5668
u'a=b\n\nc=d',
5769
[
58-
Binding(key=u"a", value=u"b", original=u"a=b\n"),
59-
Binding(key=u"c", value=u"d", original=u"\nc=d"),
70+
Binding(key=u"a", value=u"b", original=Original(string=u"a=b\n", line=1)),
71+
Binding(key=u"c", value=u"d", original=Original(string=u"\nc=d", line=2)),
6072
]
6173
),
6274
(
6375
u'a="\nb=c',
6476
[
65-
Binding(key=None, value=None, original=u'a="\n'),
66-
Binding(key=u"b", value=u"c", original=u"b=c"),
77+
Binding(key=None, value=None, original=Original(string=u'a="\n', line=1)),
78+
Binding(key=u"b", value=u"c", original=Original(string=u"b=c", line=2)),
6779
]
6880
),
6981
(
7082
u'# comment\na="b\nc"\nd=e\n',
7183
[
72-
Binding(key=None, value=None, original=u"# comment\n"),
73-
Binding(key=u"a", value=u"b\nc", original=u'a="b\nc"\n'),
74-
Binding(key=u"d", value=u"e", original=u"d=e\n"),
84+
Binding(key=None, value=None, original=Original(string=u"# comment\n", line=1)),
85+
Binding(key=u"a", value=u"b\nc", original=Original(string=u'a="b\nc"\n', line=2)),
86+
Binding(key=u"d", value=u"e", original=Original(string=u"d=e\n", line=4)),
7587
],
7688
),
7789
(
7890
u'garbage[%$#\na=b',
7991
[
80-
Binding(key=None, value=None, original=u"garbage[%$#\n"),
81-
Binding(key=u"a", value=u"b", original=u'a=b'),
92+
Binding(key=None, value=None, original=Original(string=u"garbage[%$#\n", line=1)),
93+
Binding(key=u"a", value=u"b", original=Original(string=u'a=b', line=2)),
8294
],
8395
),
8496
])

0 commit comments

Comments
 (0)