1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
|
commit 544bf807bf506927872b7b837c6e1514e80419ab
Author: Konstantin Chernyshev <k4black@ya.ru>
Date: Thu Nov 16 19:40:54 2023 +0100
fix(bleu): add Fraction with 3.12 support
diff --git a/nltk/test/unit/translate/test_bleu.py b/nltk/test/unit/translate/test_bleu.py
index 990b76406..b472d785f 100644
--- a/nltk/test/unit/translate/test_bleu.py
+++ b/nltk/test/unit/translate/test_bleu.py
@@ -2,7 +2,6 @@
Tests for BLEU translation evaluation metric
"""
-import io
import unittest
import numpy as np
diff --git a/nltk/translate/bleu_score.py b/nltk/translate/bleu_score.py
index da445bc3e..9647202e6 100644
--- a/nltk/translate/bleu_score.py
+++ b/nltk/translate/bleu_score.py
@@ -7,16 +7,40 @@
# For license information, see LICENSE.TXT
"""BLEU score implementation."""
+from __future__ import annotations
import math
import sys
import warnings
from collections import Counter
-from fractions import Fraction
+from dataclasses import dataclass
from nltk.util import ngrams
+@dataclass
+class Fraction:
+ """
+ This class is used to represent a fraction with both the numerator and denominator saved for later retrieval.
+ Python 3.12 removed _normalize=False from the standard lib Fraction constructor.
+ """
+
+ numerator: int | float
+ denominator: int = 1
+
+ def __float__(self):
+ return self.numerator / self.denominator
+
+ def __lt__(self, other):
+ return float(self) < float(other)
+
+ def __eq__(self, other):
+ return self.numerator == other.numerator and self.denominator == other.denominator
+
+ def __gt__(self, other):
+ return float(self) > float(other)
+
+
def sentence_bleu(
references,
hypothesis,
@@ -222,7 +246,7 @@ def corpus_bleu(
# Collects the various precision values for the different ngram orders.
p_n = [
- Fraction(p_numerators[i], p_denominators[i], _normalize=False)
+ Fraction(p_numerators[i], p_denominators[i])
for i in range(1, max_weight_length + 1)
]
@@ -365,7 +389,7 @@ def modified_precision(references, hypothesis, n):
# Usually this happens when the ngram order is > len(reference).
denominator = max(1, sum(counts.values()))
- return Fraction(numerator, denominator, _normalize=False)
+ return Fraction(numerator, denominator)
def closest_ref_length(references, hyp_len):
@@ -577,7 +601,7 @@ class SmoothingFunction:
In COLING 2004.
"""
return [
- Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1, _normalize=False)
+ Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1)
if i != 0
else p_n[0]
for i in range(len(p_n))
commit eab59d7ceac5dc64e34ca04ee0143fa8998af204
Author: Konstantin Chernyshev <k4black@ya.ru>
Date: Thu Nov 16 19:46:47 2023 +0100
style: fix pre-commit style checks
diff --git a/nltk/translate/bleu_score.py b/nltk/translate/bleu_score.py
index 9647202e6..aab519521 100644
--- a/nltk/translate/bleu_score.py
+++ b/nltk/translate/bleu_score.py
@@ -35,7 +35,9 @@ class Fraction:
return float(self) < float(other)
def __eq__(self, other):
- return self.numerator == other.numerator and self.denominator == other.denominator
+ return (
+ self.numerator == other.numerator and self.denominator == other.denominator
+ )
def __gt__(self, other):
return float(self) > float(other)
@@ -601,10 +603,7 @@ class SmoothingFunction:
In COLING 2004.
"""
return [
- Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1)
- if i != 0
- else p_n[0]
- for i in range(len(p_n))
+ Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1) if i != 0 else p_n[0] for i in range(len(p_n))
]
def method3(self, p_n, *args, **kwargs):
commit 8f06fa4b1da4a390d53010e56bc6ab5d5cb32a3a
Author: Konstantin Chernyshev <k4black@ya.ru>
Date: Thu Nov 16 19:48:06 2023 +0100
style: fix pre-commit style checks
diff --git a/nltk/translate/bleu_score.py b/nltk/translate/bleu_score.py
index aab519521..f32743ecc 100644
--- a/nltk/translate/bleu_score.py
+++ b/nltk/translate/bleu_score.py
@@ -603,7 +603,8 @@ class SmoothingFunction:
In COLING 2004.
"""
return [
- Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1) if i != 0 else p_n[0] for i in range(len(p_n))
+ Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1) if i != 0 else p_n[0]
+ for i in range(len(p_n))
]
def method3(self, p_n, *args, **kwargs):
commit b72a4bdbaf99a2bdb6a99454a1eb8804051bf644
Author: Konstantin Chernyshev <k4black@ya.ru>
Date: Thu Nov 16 21:49:32 2023 +0100
fix: Fraction
diff --git a/nltk/translate/bleu_score.py b/nltk/translate/bleu_score.py
index f32743ecc..72f5b548d 100644
--- a/nltk/translate/bleu_score.py
+++ b/nltk/translate/bleu_score.py
@@ -42,6 +42,14 @@ class Fraction:
def __gt__(self, other):
return float(self) > float(other)
+ def __add__(self, other):
+ if isinstance(other, int):
+ other = Fraction(other, 1)
+ return Fraction(
+ self.numerator * other.denominator + other.numerator * self.denominator,
+ self.denominator * other.denominator,
+ )
+
def sentence_bleu(
references,
commit 86fa0832f0f4b366f96867f59ae05d744d68b513
Author: Konstantin Chernyshev <k4black@ya.ru>
Date: Thu Nov 16 23:26:42 2023 +0100
fix: use Fraction override
diff --git a/nltk/translate/bleu_score.py b/nltk/translate/bleu_score.py
index 72f5b548d..ddb54dba8 100644
--- a/nltk/translate/bleu_score.py
+++ b/nltk/translate/bleu_score.py
@@ -7,48 +7,39 @@
# For license information, see LICENSE.TXT
"""BLEU score implementation."""
-from __future__ import annotations
-
import math
import sys
import warnings
from collections import Counter
-from dataclasses import dataclass
+from fractions import Fraction as _Fraction
from nltk.util import ngrams
-@dataclass
-class Fraction:
- """
- This class is used to represent a fraction with both the numerator and denominator saved for later retrieval.
- Python 3.12 removed _normalize=False from the standard lib Fraction constructor.
- """
-
- numerator: int | float
- denominator: int = 1
-
- def __float__(self):
- return self.numerator / self.denominator
-
- def __lt__(self, other):
- return float(self) < float(other)
+class Fraction(_Fraction):
+ """Fraction with _normalize=False support for 3.12"""
- def __eq__(self, other):
- return (
- self.numerator == other.numerator and self.denominator == other.denominator
- )
+ def __new__(cls, numerator=0, denominator=None, _normalize=False):
+ if sys.version_info >= (3, 12):
+ self = super().__new__(cls, numerator, denominator)
+ else:
+ self = super().__new__(cls, numerator, denominator, _normalize=_normalize)
+ self._normalize = _normalize
+ self._original_numerator = numerator
+ self._original_denominator = denominator
+ return self
- def __gt__(self, other):
- return float(self) > float(other)
+ @property
+ def numerator(self):
+ if self._normalize:
+ return self._numerator
+ return self._original_numerator
- def __add__(self, other):
- if isinstance(other, int):
- other = Fraction(other, 1)
- return Fraction(
- self.numerator * other.denominator + other.numerator * self.denominator,
- self.denominator * other.denominator,
- )
+ @property
+ def denominator(self):
+ if self._normalize:
+ return self._denominator
+ return self._original_denominator
def sentence_bleu(
@@ -256,7 +247,7 @@ def corpus_bleu(
# Collects the various precision values for the different ngram orders.
p_n = [
- Fraction(p_numerators[i], p_denominators[i])
+ Fraction(p_numerators[i], p_denominators[i], _normalize=False)
for i in range(1, max_weight_length + 1)
]
@@ -399,7 +390,7 @@ def modified_precision(references, hypothesis, n):
# Usually this happens when the ngram order is > len(reference).
denominator = max(1, sum(counts.values()))
- return Fraction(numerator, denominator)
+ return Fraction(numerator, denominator, _normalize=False)
def closest_ref_length(references, hyp_len):
@@ -611,7 +602,8 @@ class SmoothingFunction:
In COLING 2004.
"""
return [
- Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1) if i != 0 else p_n[0]
+ Fraction(p_n[i].numerator + 1, p_n[i].denominator + 1, _normalize=False)
+ if i != 0 else p_n[0]
for i in range(len(p_n))
]
|
