estym/m2-traitement-benchmark
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fixed shard creations

Browse source
This commit is contained in:
Evann Regnault 2024-06-01 17:12:31 +02:00
parent 7f0d8a7dda
commit c6dc0a797b
4 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
drivers/elastic.py
View file

@ -43,6 +43,7 @@ class Elastic_Testing(DB_Testing):
driver.indices.delete(index=INDEX) driver.indices.delete(index=INDEX)
def add_bulk(self, driver: Elasticsearch): def add_bulk(self, driver: Elasticsearch):
self.create_table(driver)
for i in range(0, len(self.bulk_data), 1000): for i in range(0, len(self.bulk_data), 1000):
driver.bulk(index=INDEX, operations = '\n'.join(self.bulk_data[i:i+1000]), refresh=True) driver.bulk(index=INDEX, operations = '\n'.join(self.bulk_data[i:i+1000]), refresh=True)

4
main.py
View file

@ -22,8 +22,8 @@ def test_psql():
def test_elasticsearch(shards = 1): def test_elasticsearch(shards = 1):
es = Elasticsearch( es = Elasticsearch(
'https://127.0.0.1:9200', 'http://docker.tma.coe.int:9200',
api_key='RVAzeHpvOEJXTnUzZ2RiTkdWX2Q6TksySHBfaEFSWktoQmNPOFFSbm1DUQ==', api_key='YVE1VDA0OEI3c28wRGJYTUZkLW46OWFYSUQtQVhUcGVvTmlKdUtmS29Wdw==',
verify_certs=False, # just to not create certificates verify_certs=False, # just to not create certificates
ssl_show_warn=False ssl_show_warn=False
) )

BIN
plot-orodruin-opti.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 73 KiB

After

Width:  |  Height:  |  Size: 71 KiB

4
utils/utils.py
View file

@ -26,11 +26,11 @@ def time_func(name: str, function_name: str, x : Callable, repeat : Optional[int
def preprocess_json(): def preprocess_json():
for x in range(0, 1): for x in range(0, 10):
with open('data.json', 'r') as f: with open('data.json', 'r') as f:
data : dict = json.load(f) data : dict = json.load(f)
for d in data: for d in data:
x = Data_Thread() x = Data_Thread()
for k in d.keys(): for k in d.keys():
x.__setattr__(k, d[k]) x.__setattr__(k, d[k])
Threads.append(x) Threads.append(x)