Skip to content
Snippets Groups Projects
Verified Commit f57ee93e authored by Daniel Hornung's avatar Daniel Hornung
Browse files

MAINT: Renaming split_into_inserts_and_updates.

parent c22af986
Branches
Tags
2 merge requests!178FIX: #96 Better error output for crawl.py script.,!171sav/spss converter
Pipeline #51983 failed
...@@ -295,11 +295,17 @@ class Crawler(object): ...@@ -295,11 +295,17 @@ class Crawler(object):
self.crawled_data = data self.crawled_data = data
return data return data
def split_into_inserts_and_updates(self, st: SyncGraph): def _split_into_inserts_and_updates(self, st: SyncGraph):
"""Classify nodes in the SyncGraph ``st`` with respect to their state on the server. """Classify nodes in the SyncGraph ``st`` with respect to their state on the server.
This method iteratively checks whether those nodes exist on the remote server and creates two lists, This method iteratively checks whether those nodes exist on the remote server and creates two lists,
one with the entities that need to be updated and the other with entities to be inserted. one with the entities that need to be updated and the other with entities to be inserted.
.. todo::
Should this be made into a public method of SyncGraph instead? At the moment, this is a
purely static method that only operates on the state of ``st``.
""" """
entity_was_treated = True entity_was_treated = True
# st.unchecked contains Entities which could not yet be checked against the remote server # st.unchecked contains Entities which could not yet be checked against the remote server
...@@ -338,7 +344,7 @@ one with the entities that need to be updated and the other with entities to be ...@@ -338,7 +344,7 @@ one with the entities that need to be updated and the other with entities to be
# ) # )
raise RuntimeError( raise RuntimeError(
"Could not finish split_into_inserts_and_updates. " "Could not finish _split_into_inserts_and_updates. "
"It might be due to a circular dependency") "It might be due to a circular dependency")
return st.export_record_lists() return st.export_record_lists()
...@@ -650,7 +656,7 @@ one with the entities that need to be updated and the other with entities to be ...@@ -650,7 +656,7 @@ one with the entities that need to be updated and the other with entities to be
"use for example the Scanner to create this data.")) "use for example the Scanner to create this data."))
crawled_data = self.crawled_data crawled_data = self.crawled_data
to_be_inserted, to_be_updated = self.split_into_inserts_and_updates( to_be_inserted, to_be_updated = self._split_into_inserts_and_updates(
SyncGraph(crawled_data, self.identifiableAdapter)) SyncGraph(crawled_data, self.identifiableAdapter))
for el in to_be_updated: for el in to_be_updated:
......
...@@ -329,7 +329,7 @@ def test_remove_unnecessary_updates(): ...@@ -329,7 +329,7 @@ def test_remove_unnecessary_updates():
def test_split_into_inserts_and_updates_trivial(): def test_split_into_inserts_and_updates_trivial():
crawler = Crawler() crawler = Crawler()
st = SyncGraph([], crawler.identifiableAdapter) st = SyncGraph([], crawler.identifiableAdapter)
crawler.split_into_inserts_and_updates(st) crawler._split_into_inserts_and_updates(st)
def test_split_into_inserts_and_updates_simple(crawler_mocked_identifiable_retrieve): def test_split_into_inserts_and_updates_simple(crawler_mocked_identifiable_retrieve):
...@@ -347,7 +347,7 @@ def test_split_into_inserts_and_updates_simple(crawler_mocked_identifiable_retri ...@@ -347,7 +347,7 @@ def test_split_into_inserts_and_updates_simple(crawler_mocked_identifiable_retri
assert crawler.identifiableAdapter.retrieve_identified_record_for_record( assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
identlist[1]) is None identlist[1]) is None
insert, update = crawler.split_into_inserts_and_updates(st) insert, update = crawler._split_into_inserts_and_updates(st)
assert len(insert) == 1 assert len(insert) == 1
assert insert[0].name == "B" assert insert[0].name == "B"
assert len(update) == 1 assert len(update) == 1
...@@ -370,7 +370,7 @@ def test_split_into_inserts_and_updates_with_circ(crawler_mocked_identifiable_re ...@@ -370,7 +370,7 @@ def test_split_into_inserts_and_updates_with_circ(crawler_mocked_identifiable_re
st = SyncGraph([a, b], crawler.identifiableAdapter) st = SyncGraph([a, b], crawler.identifiableAdapter)
with pytest.raises(RuntimeError): with pytest.raises(RuntimeError):
crawler.split_into_inserts_and_updates(st) crawler._split_into_inserts_and_updates(st)
def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable_retrieve): def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable_retrieve):
...@@ -389,7 +389,7 @@ def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable ...@@ -389,7 +389,7 @@ def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable
b.add_property("C", f) b.add_property("C", f)
entlist = [a, b, g] entlist = [a, b, g]
st = SyncGraph(entlist, crawler.identifiableAdapter) st = SyncGraph(entlist, crawler.identifiableAdapter)
insert, update = crawler.split_into_inserts_and_updates(st) insert, update = crawler._split_into_inserts_and_updates(st)
assert len(insert) == 3 assert len(insert) == 3
assert "B" in [el.name for el in insert] assert "B" in [el.name for el in insert]
assert len(update) == 1 assert len(update) == 1
...@@ -487,7 +487,7 @@ a: ([b1, b2]) ...@@ -487,7 +487,7 @@ a: ([b1, b2])
# The Cs cannot be merged due to different identifying properties # The Cs cannot be merged due to different identifying properties
# The Bs cannot be merged due to different references to Cs # The Bs cannot be merged due to different references to Cs
with raises(ImpossibleMergeError) as rte: with raises(ImpossibleMergeError) as rte:
crawler.split_into_inserts_and_updates(st) crawler._split_into_inserts_and_updates(st)
# TODO # TODO
# assert not isinstance(rte.value, NotImplementedError), \ # assert not isinstance(rte.value, NotImplementedError), \
# "Exception must not be NotImplementedError, but plain RuntimeError." # "Exception must not be NotImplementedError, but plain RuntimeError."
...@@ -521,7 +521,7 @@ def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test) ...@@ -521,7 +521,7 @@ def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test)
identlist[1]) is None identlist[1]) is None
# check the split... # check the split...
insert, update = crawler.split_into_inserts_and_updates(st) insert, update = crawler._split_into_inserts_and_updates(st)
# A was found remotely and is therefore in the update list # A was found remotely and is therefore in the update list
assert len(update) == 1 assert len(update) == 1
assert update[0].name == "A" assert update[0].name == "A"
...@@ -550,7 +550,7 @@ def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_ ...@@ -550,7 +550,7 @@ def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_
assert len(identifiable.backrefs) == 2 assert len(identifiable.backrefs) == 2
# check the split... # check the split...
insert, update = crawler.split_into_inserts_and_updates(st) insert, update = crawler._split_into_inserts_and_updates(st)
assert len(update) == 2 assert len(update) == 2
assert len(insert) == 1 assert len(insert) == 1
...@@ -575,7 +575,7 @@ def test_split_into_inserts_and_updates_diff_backref(crawler_mocked_for_backref_ ...@@ -575,7 +575,7 @@ def test_split_into_inserts_and_updates_diff_backref(crawler_mocked_for_backref_
assert len(identifiable.backrefs) == 2 assert len(identifiable.backrefs) == 2
# check the split... # check the split...
insert, update = crawler.split_into_inserts_and_updates(st) insert, update = crawler._split_into_inserts_and_updates(st)
assert len(update) == 2 assert len(update) == 2
assert len(insert) == 1 assert len(insert) == 1
......
...@@ -18,6 +18,8 @@ ...@@ -18,6 +18,8 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
# #
import logging
from functools import partial from functools import partial
from unittest.mock import MagicMock, Mock, patch from unittest.mock import MagicMock, Mock, patch
...@@ -631,8 +633,8 @@ def test_detect_circular_dependency(crawler_mocked_identifiable_retrieve, caplog ...@@ -631,8 +633,8 @@ def test_detect_circular_dependency(crawler_mocked_identifiable_retrieve, caplog
assert Crawler.detect_circular_dependency([d]) is None assert Crawler.detect_circular_dependency([d]) is None
st = SyncGraph(flat, crawler.identifiableAdapter) st = SyncGraph(flat, crawler.identifiableAdapter)
with raises(RuntimeError): with pytest.raises(RuntimeError):
_, _ = crawler.split_into_inserts_and_updates(st) _, _ = crawler._split_into_inserts_and_updates(st)
caplog.set_level(logging.ERROR, logger="caoscrawler.converters") caplog.set_level(logging.ERROR, logger="caoscrawler.converters")
assert "Found circular dependency" in caplog.text assert "Found circular dependency" in caplog.text
assert "\n--------\n\n> Parent: C\n\n>> Name: a\n[\'C\']" in caplog.text assert "\n--------\n\n> Parent: C\n\n>> Name: a\n[\'C\']" in caplog.text
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment