Unverified Commit 729b2bf8 authored by Rob Speer's avatar Rob Speer Committed by GitHub
Browse files

Merge pull request #193 from commonsense/faster-evaluate

Faster evaluate
parents f11c4136 8bf59f97
......@@ -136,9 +136,10 @@ def run_debias(input_filename, output_filename):
@click.argument('filename', type=click.Path(readable=True, dir_okay=False))
@click.option('--subset', '-s', type=click.Choice(['dev', 'test', 'all']), default='dev')
@click.option('--semeval-by-language/--semeval-global', '-l', default=False)
def run_evaluate(filename, subset, semeval_by_language):
@click.option('--run-analogies', is_flag=True)
def run_evaluate(filename, subset, semeval_by_language, run_analogies):
"""
Evaluate a frame on word similarity and analogy tasks. Measure its bias.
Evaluate a frame on word similarity and (optionally) analogy tasks. Measure its bias.
"""
frame = load_hdf(filename)
if semeval_by_language:
......@@ -146,7 +147,8 @@ def run_evaluate(filename, subset, semeval_by_language):
else:
scope = 'global'
print(wordsim.evaluate(frame, subset=subset, semeval_scope=scope))
print(analogy.evaluate(frame, subset=subset, analogy_filename=ANALOGY_FILENAME))
if run_analogies:
print(analogy.evaluate(frame, subset=subset, analogy_filename=ANALOGY_FILENAME))
print(bias.measure_bias(frame))
......@@ -206,7 +208,8 @@ def run_evaluate_bias(filename):
@cli.command(name='compare_embeddings')
@click.argument('input_filenames', nargs=-1, type=click.Path(readable=True, dir_okay=False))
@click.argument('output_filename', type=click.Path(writable=True, dir_okay=False))
def run_compare_embeddings(input_filenames, output_filename):
@click.option('--run-analogies', is_flag=True)
def run_compare_embeddings(input_filenames, output_filename, run_analogies):
"""
The `input_filenames` are files that can be loaded as matrices of word
embeddings. They'll be run through the relatedness and analogy evaluations,
......@@ -218,7 +221,7 @@ def run_compare_embeddings(input_filenames, output_filename):
neighbors in the ConceptNet graph. These embeddings could have been stored
in the matrix, but this saves memory and download time.
"""
results = compare_embeddings(input_filenames, subset='all', tune_analogies=True)
results = compare_embeddings(input_filenames, subset='all', run_analogies=run_analogies)
print(results)
save_hdf(results, output_filename)
......
......@@ -500,7 +500,7 @@ def eval_bats_category(vectors, category, vocab_size=200000, verbose=False):
return category_results
def evaluate(frame, analogy_filename, subset='test', tune_analogies=False, scope='global',
def evaluate(frame, analogy_filename, subset='test', tune_analogies=True, scope='global',
google_vocab_size=200000):
"""
Run SAT and Semeval12-2 evaluations.
......@@ -514,13 +514,12 @@ def evaluate(frame, analogy_filename, subset='test', tune_analogies=False, scope
Optional parameters:
subset (string, default 'test')
a subset of a data to evaluate on, either 'test' or 'dev'
tune_analogies (boolean, default False)
tune_analogies (boolean, default True)
tune the weights in eval_pairwise_analogies()
semeval_scope (string, default 'global')
'global' to get the average of the results across all subclasses of semeval12-2,
or another string to get the results broken down by a subclass (1a, 1b, etc.)
"""
vectors = VectorSpaceWrapper(frame=frame)
results = empty_comparison_table()
......
......@@ -21,18 +21,22 @@ def load_any_embeddings(filename):
raise ValueError("Can't recognize file extension of %r" % filename)
def compare_embeddings(filenames, subset='dev', tune_analogies=False):
def compare_embeddings(filenames, subset='dev', run_analogies=False):
results = []
for filename in filenames:
print(filename)
frame = load_any_embeddings(filename)
wordsim_results = wordsim.evaluate(frame, subset=subset, semeval_scope='per-language')
analogy_results = analogy.evaluate(frame, ANALOGY_FILENAME, tune_analogies=tune_analogies)
story_results = story.evaluate(frame, subset=subset).to_frame('story-cloze').T
frame_results = [wordsim_results, story_results]
if run_analogies:
analogy_results = analogy.evaluate(frame, ANALOGY_FILENAME, subset=subset)
frame_results.append(analogy_results)
results.append(
pd.concat(
[wordsim_results, analogy_results, story_results], axis=0
frame_results, axis=0
)
)
result = pd.concat(results, keys=filenames)
......
......@@ -64,6 +64,7 @@ class VectorSpaceWrapper(object):
self.small_k = None
self.finder = None
self.trie = None
self.cache = {}
if use_db:
self.finder = AssertionFinder()
......@@ -198,6 +199,11 @@ class VectorSpaceWrapper(object):
will allow expanded_vector to look up neighboring terms in ConceptNet.
"""
self.load()
query_index = tuple(query)
if query_index in self.cache:
return self.cache[query_index]
if isinstance(query, np.ndarray):
return query
elif isinstance(query, pd.Series) or isinstance(query, dict):
......@@ -212,7 +218,8 @@ class VectorSpaceWrapper(object):
raise ValueError("Can't make a query out of type %s" % type(query))
include_neighbors = include_neighbors and (len(terms) <= 5)
vec = self.expanded_vector(terms, include_neighbors=include_neighbors)
return normalize_vec(vec)
self.cache[query_index] = normalize_vec(vec)
return self.cache[query_index]
def similar_terms(self, query, filter=None, limit=20):
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment