Skip to content

Commit 32b1f0f

Browse files
author
livecodeali
committed
[[ Bug 16161 ]] Word chunk should be delimited by lineDel, not itemDel
1 parent 1724560 commit 32b1f0f

6 files changed

Lines changed: 92 additions & 11 deletions

File tree

‎docs/notes/bugfix-16161.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Word chunk is erroneously delimited by item delimiter

‎engine/src/exec-strings-chunk.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1182,7 +1182,7 @@ MCTextChunkIterator *MCStringsTextChunkIteratorCreate(MCExecContext& ctxt, MCStr
11821182
return tci;
11831183
}
11841184

1185-
return MCChunkCreateTextChunkIterator(p_text, nil, MCChunkTypeFromChunkTerm(p_chunk_type), p_chunk_type == CT_LINE ? ctxt . GetLineDelimiter() : ctxt . GetItemDelimiter(), ctxt . GetStringComparisonType());
1185+
return MCChunkCreateTextChunkIterator(p_text, nil, MCChunkTypeFromChunkTerm(p_chunk_type), ctxt . GetLineDelimiter(), ctxt . GetItemDelimiter(), ctxt . GetStringComparisonType());
11861186
}
11871187

11881188
MCTextChunkIterator *MCStringsTextChunkIteratorCreateWithRange(MCExecContext& ctxt, MCStringRef p_text, MCRange p_range, Chunk_term p_chunk_type)
@@ -1194,7 +1194,7 @@ MCTextChunkIterator *MCStringsTextChunkIteratorCreateWithRange(MCExecContext& ct
11941194
return tci;
11951195
}
11961196

1197-
return MCChunkCreateTextChunkIterator(p_text, &p_range, MCChunkTypeFromChunkTerm(p_chunk_type), p_chunk_type == CT_LINE ? ctxt . GetLineDelimiter() : ctxt . GetItemDelimiter(), ctxt . GetStringComparisonType());
1197+
return MCChunkCreateTextChunkIterator(p_text, &p_range, MCChunkTypeFromChunkTerm(p_chunk_type), ctxt . GetLineDelimiter(), ctxt . GetItemDelimiter(), ctxt . GetStringComparisonType());
11981198
}
11991199

12001200
bool MCStringsTextChunkIteratorNext(MCExecContext& ctxt, MCTextChunkIterator *tci)

‎libfoundation/include/foundation-chunk.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,6 @@ class MCTextChunkIterator_Tokenized : public MCTextChunkIterator
284284
virtual bool Next();
285285
};
286286

287-
MCTextChunkIterator *MCChunkCreateTextChunkIterator(MCStringRef p_text, MCRange *p_range, MCChunkType p_chunk_type, MCStringRef p_delimiter, MCStringOptions p_options);
287+
MCTextChunkIterator *MCChunkCreateTextChunkIterator(MCStringRef p_text, MCRange *p_range, MCChunkType p_chunk_type, MCStringRef p_line_delimiter, MCStringRef p_item_delimiter, MCStringOptions p_options);
288288

289289
#endif // __MC_FOUNDATION_CHUNK__

‎libfoundation/src/foundation-chunk.cpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -892,7 +892,7 @@ bool MCTextChunkIterator_Word::Next()
892892

893893
////////////////////////////////////////////////////////////////////////////////
894894

895-
MCTextChunkIterator *MCChunkCreateTextChunkIterator(MCStringRef p_text, MCRange *p_range, MCChunkType p_chunk_type, MCStringRef p_delimiter, MCStringOptions p_options)
895+
MCTextChunkIterator *MCChunkCreateTextChunkIterator(MCStringRef p_text, MCRange *p_range, MCChunkType p_chunk_type, MCStringRef p_line_delimiter, MCStringRef p_item_delimiter, MCStringOptions p_options)
896896
{
897897
if (p_chunk_type == kMCChunkTypeCharacter && (MCStringIsNative(p_text) || (MCStringIsSimple(p_text) && MCStringIsUncombined(p_text))))
898898
p_chunk_type = kMCChunkTypeCodeunit;
@@ -911,10 +911,16 @@ MCTextChunkIterator *MCChunkCreateTextChunkIterator(MCStringRef p_text, MCRange
911911
break;
912912
case kMCChunkTypeLine:
913913
case kMCChunkTypeItem:
914+
MCStringRef t_delimiter;
915+
if (p_chunk_type == kMCChunkTypeLine)
916+
t_delimiter = p_line_delimiter;
917+
else
918+
t_delimiter = p_item_delimiter;
919+
914920
if (p_range != nil)
915-
t_iterator = new MCTextChunkIterator_Delimited(p_text, p_chunk_type, p_delimiter, *p_range);
921+
t_iterator = new MCTextChunkIterator_Delimited(p_text, p_chunk_type, t_delimiter, *p_range);
916922
else
917-
t_iterator = new MCTextChunkIterator_Delimited(p_text, p_chunk_type, p_delimiter);
923+
t_iterator = new MCTextChunkIterator_Delimited(p_text, p_chunk_type, t_delimiter);
918924
break;
919925

920926
break;
@@ -926,10 +932,12 @@ MCTextChunkIterator *MCChunkCreateTextChunkIterator(MCStringRef p_text, MCRange
926932
break;
927933
break;
928934
case kMCChunkTypeWord:
935+
// AL-2015-10-08: [[ Bug 16161 ]] Word chunk needs to be passed line delimiter
936+
// as words are also delimited by line breaks.
929937
if (p_range != nil)
930-
t_iterator = new MCTextChunkIterator_Word(p_text, p_chunk_type, p_delimiter, *p_range);
938+
t_iterator = new MCTextChunkIterator_Word(p_text, p_chunk_type, p_line_delimiter, *p_range);
931939
else
932-
t_iterator = new MCTextChunkIterator_Word(p_text, p_chunk_type, p_delimiter);
940+
t_iterator = new MCTextChunkIterator_Word(p_text, p_chunk_type, p_line_delimiter);
933941
break;
934942
case kMCChunkTypeCodepoint:
935943
if (p_range != nil)

‎libscript/src/module-char.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ bool MCCharStoreChunk(MCStringRef &x_target, MCStringRef p_value, MCRange p_grap
5050
extern "C" MC_DLLEXPORT_DEF void MCCharEvalNumberOfCharsIn(MCStringRef p_target, index_t& r_output)
5151
{
5252
MCTextChunkIterator *tci;
53-
tci = MCChunkCreateTextChunkIterator(p_target, nil, kMCChunkTypeCharacter, nil, kMCStringOptionCompareExact);
53+
tci = MCChunkCreateTextChunkIterator(p_target, nil, kMCChunkTypeCharacter, nil, nil, kMCStringOptionCompareExact);
5454
r_output = tci -> CountChunks();
5555
}
5656

@@ -67,7 +67,7 @@ extern "C" MC_DLLEXPORT_DEF void MCCharEvalIsAmongTheCharsOf(MCStringRef p_needl
6767
}
6868

6969
MCTextChunkIterator *tci;
70-
tci = MCChunkCreateTextChunkIterator(p_target, nil, kMCChunkTypeCharacter, nil, kMCStringOptionCompareExact);
70+
tci = MCChunkCreateTextChunkIterator(p_target, nil, kMCChunkTypeCharacter, nil, nil, kMCStringOptionCompareExact);
7171
r_output = tci -> IsAmong(p_needle);
7272
}
7373

@@ -255,7 +255,7 @@ extern "C" MC_DLLEXPORT_DEF bool MCCharRepeatForEachChar(void*& x_iterator, MCSt
255255
if ((uintptr_t)x_iterator == 0)
256256
{
257257
t_first = true;
258-
t_iterator = MCChunkCreateTextChunkIterator(p_string, nil, kMCChunkTypeCharacter, nil, kMCStringOptionCompareExact);
258+
t_iterator = MCChunkCreateTextChunkIterator(p_string, nil, kMCChunkTypeCharacter, nil, nil, kMCStringOptionCompareExact);
259259
}
260260
else
261261
t_iterator = (MCTextChunkIterator *)x_iterator;

‎tests/lcs/core/chunks/segment.livecodescript‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
script "CoreChunksSegment"
2+
/*
3+
Copyright (C) 2015 Runtime Revolution Ltd.
4+
5+
This file is part of LiveCode.
6+
7+
LiveCode is free software; you can redistribute it and/or modify it under
8+
the terms of the GNU General Public License v3 as published by the Free
9+
Software Foundation.
10+
11+
LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
12+
WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
for more details.
15+
16+
You should have received a copy of the GNU General Public License
17+
along with LiveCode. If not see <http://www.gnu.org/licenses/>. */
18+
19+
on TestSegmentRanges
20+
TestAssert "overlong prefix", segment -4 to -2 of "three segment sentence" is "three segment"
21+
TestAssert "overlong suffix", segment 2 to 4 of "three segment sentence" is "segment sentence"
22+
end TestSegmentRanges
23+
24+
on TestSegmentRangesEmpty
25+
TestAssert "empty prefix", segment -5 to -4 of "three segment sentence" is ""
26+
TestAssert "empty suffix", segment 4 to 5 of "three segment sentence" is ""
27+
TestAssert "empty range", segment 5 to 4 of "three segment sentence" is ""
28+
end TestSegmentRangesEmpty
29+
30+
on TestSegmentRangesSingle
31+
TestAssert "single segment range", segment 1 to 1 of "three segment sentence" is "three"
32+
TestAssert "single segment range from back", segment -2 to -2 of "three segment sentence" is "segment"
33+
end TestSegmentRangesSingle
34+
35+
on TestSegmentExpressions
36+
TestAssert "single segment", segment 1 of "three segment sentence" is "three"
37+
TestAssert "single segment from back", segment -3 of "three segment sentence" is "three"
38+
end TestSegmentExpressions
39+
40+
on TestSegmentExpressionsQuoted
41+
local tVar
42+
put quote & "first segment" & quote && "and" && quote & "third segment" & quote into tVar
43+
TestAssert "single segment quoted", segment 2 of tVar is "and"
44+
TestAssert "single segment quoted from back", segment -2 of tVar is "and"
45+
end TestSegmentExpressionsQuoted
46+
47+
on TestSegmentExpressionsEmpty
48+
TestAssert "single segment", segment 4 of "three segment sentence" is ""
49+
TestAssert "single segment", segment -4 of "three segment sentence" is ""
50+
end TestSegmentExpressionsEmpty
51+
52+
on TestSegmentLineBreakCount
53+
TestAssert "count with line break", the number of segments in ("one" & return & "two") is 2
54+
TestAssert "count with line break quoted", the number of segments in (quote & "one" & return & "two" & quote) is 2
55+
end TestSegmentLineBreakCount
56+
57+
on TestSegmentLineBreakExpression
58+
TestAssert "single segment", segment 2 of ("one" & return & "two") is "two"
59+
TestAssert "single segment quoted", segment 2 of (quote & "one" & return & "two" & quote) is ("two" & quote)
60+
end TestSegmentLineBreakExpression
61+
62+
on TestSegmentCount
63+
TestAssert "count", the number of segments in "three segment sentence" is 3
64+
TestAssert "count", the number of segments in (quote & "one segment sentence" & quote) is 1
65+
end TestSegmentCount
66+
67+
on TestBug16161
68+
local tLine, tExpected
69+
put "assert true (comma is" && quote & "," & quote && "AND comma is numToChar(44))" into tLine
70+
put "(comma is" && quote & "," & quote && "AND comma is numToChar(44))" into tExpected
71+
TestAssert "comma is not word delimiter", word 3 to -1 of tLine is tExpected
72+
end TestBug16161

0 commit comments

Comments
 (0)