Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Kizito Ononuju (Kiz')
conceptnet5
Commits
65f0c03d
Commit
65f0c03d
authored
Feb 07, 2019
by
Robyn Speer
Browse files
enable blacklisting particular assertions
parent
77993468
Changes
2
Hide whitespace changes
Inline
Side-by-side
conceptnet5/builders/combine_assertions.py
View file @
65f0c03d
...
...
@@ -11,11 +11,17 @@ from conceptnet5.languages import ALL_LANGUAGES
from
conceptnet5.readers.wiktionary
import
valid_language
from
conceptnet5.uri
import
conjunction_uri
,
get_uri_language
,
is_absolute_url
,
Licenses
,
\
split_uri
,
uri_prefix
from
conceptnet5.util
import
get_support_data_filename
N
=
100
CURRENT_DIR
=
os
.
getcwd
()
def
get_blacklist
():
filename
=
get_support_data_filename
(
'blacklist.txt'
)
return
set
(
open
(
filename
).
readlines
())
def
weight_scale
(
weight
):
"""
This scale starts out linear, then switches to a square-root scale at x=2.
...
...
@@ -119,15 +125,19 @@ def combine_assertions(input_filename, output_filename):
out
=
MsgpackStreamWriter
(
output_filename
)
out_bad
=
MsgpackStreamWriter
(
output_filename
+
'.reject'
)
blacklist
=
get_blacklist
()
with
open
(
input_filename
,
encoding
=
'utf-8'
)
as
stream
:
for
key
,
line_group
in
itertools
.
groupby
(
stream
,
group_func
):
assertion
=
make_assertion
(
line_group
)
destination
=
out
if
assertion
is
None
:
continue
if
assertion
[
'weight'
]
>
0
:
destination
=
out
else
:
if
assertion
[
'weight'
]
<=
0
:
destination
=
out_bad
for
value
in
assertion
.
values
():
if
isinstance
(
value
,
str
)
and
value
in
blacklist
:
destination
=
out_bad
destination
.
write
(
assertion
)
out
.
close
()
...
...
conceptnet5/support_data/blacklist.txt
0 → 100644
View file @
65f0c03d
# We should be able to address particular bad assertions by removing them.
# Here's one to start with.
/a/[/r/HasSubevent/,/c/en/get_drunk/,/c/en/drive_carefully/]
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment