From 223dfce755ca2214737b30bc9cf6b67e37ea0cc8 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Sat, 1 Jul 2023 21:24:46 -0400 Subject: [PATCH 01/11] Update Setup.py to be compatible with latest PyTorch --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 281c337..30785d7 100644 --- a/setup.py +++ b/setup.py @@ -7,15 +7,15 @@ name="biobert-embedding", packages=['biobert_embedding'], version="0.1.2", - author="Jitendra Jangid", - author_email="jitujangid38@gmail.com", + author="Jitendra Jangid, Ariel Lubonja", + author_email="jitujangid38@gmail.com, ariellubonja@live.com", description="Embeddings from BioBERT", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/overfitter/biobert_embedding", download_url="https://github.com/overfitter/biobert_embedding/archive/v0.1.2.tar.gz", install_requires=[ - 'torch==1.2.0', + 'torch==1.13.1', 'pytorch-pretrained-bert==0.6.2', 'tensorflow', ], From 30d865efec7f41d5231c04db5f69ca8192b006f6 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Sat, 1 Jul 2023 21:25:48 -0400 Subject: [PATCH 02/11] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4f4c387..a3a6ffc 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +Ariel: Minor updates to code to make it install with pip as of Jul-2023 + # BioBert Embeddings Token and sentence level embeddings from BioBERT model (Biomedical Domain). From af7c152b53186afbfa8c989dc747176c412d7e7c Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Sat, 1 Jul 2023 22:25:26 -0400 Subject: [PATCH 03/11] Many changes to Downloader function. Now downloading from HuggingFace --- biobert_embedding/downloader.py | 79 --------------------------------- biobert_embedding/embedding.py | 40 +++++++++++++---- setup.py | 7 ++- 3 files changed, 35 insertions(+), 91 deletions(-) delete mode 100644 biobert_embedding/downloader.py diff --git a/biobert_embedding/downloader.py b/biobert_embedding/downloader.py deleted file mode 100644 index f0ac68b..0000000 --- a/biobert_embedding/downloader.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import tarfile -import requests -import tensorflow as tf -from pathlib import Path - -dropbox_id = "https://www.dropbox.com/s/hvsemunmv0htmdk/biobert_v1.1_pubmed_pytorch_model.tar.gz?dl=0" -gdd_id = "1TFtdE5pu0LiFTD4p7NEESwyVbhrY2_04" - -def download_file_from_google_drive(id, destination): - - URL = "https://docs.google.com/uc?export=download" - session = requests.Session() - - response = session.get(URL, params = { 'id' : id }, stream = True, verify = False) - token = get_confirm_token(response) - - if token: - params = { 'id' : id, 'confirm' : token } - response = session.get(URL, params = params, stream = True, verify = False) - - save_response_content(response, destination) - -def get_confirm_token(response): - for key, value in response.cookies.items(): - if key.startswith('download_warning'): - return value - - return None - -def save_response_content(response, destination): - CHUNK_SIZE = 32768 - - with open(destination, "wb") as f: - for chunk in response.iter_content(CHUNK_SIZE): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - -def get_BioBert(location): - - model_path = Path.cwd()/'biobert_v1.1_pubmed_pytorch_model' - - if location == 'dropbox': - if not os.path.isdir(model_path): - os.makedirs(model_path) - dataset = tf.keras.utils.get_file(fname=model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz", - origin = dropbox_id) - tar = tarfile.open(model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz") - tar.extractall() - - else: - if not os.path.exists(model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz"): - dataset = tf.keras.utils.get_file(fname=model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz", - origin=dropbox_id) - if not os.path.exists(model_path/"pytorch_model.bin"): - print("Extracting biobert model tar.gz") - tar = tarfile.open(model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz") - tar.extractall(model_path) - - if location == 'google drive': - if not os.path.isdir(model_path): - os.makedirs(model_path) - print("Downloading the biobert model, will take a minute...") - download_file_from_google_drive(gdd_id, model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz") - tar = tarfile.open(model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz") - tar.extractall(model_path) - else: - if not os.path.exists(model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz"): - download_file_from_google_drive(gdd_id, model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz") - if not os.path.exists(model_path/"pytorch_model.bin"): - print("Extracting biobert model tar.gz") - tar = tarfile.open(model_path/"biobert_v1.1_pubmed_pytorch_model.tar.gz") - tar.extractall(model_path) - - return model_path - - -if __name__ == "__main__": - print("package from downloading biobert model") diff --git a/biobert_embedding/embedding.py b/biobert_embedding/embedding.py index 910cc72..1f664ba 100644 --- a/biobert_embedding/embedding.py +++ b/biobert_embedding/embedding.py @@ -1,12 +1,14 @@ import os import torch import logging -import tensorflow as tf -from pathlib import Path -from biobert_embedding import downloader +import requests +from tqdm import tqdm +from urllib.parse import urlparse from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM -__author__ = 'Jitendra Jangid' +__author__ = 'Jitendra Jangid, Ariel Lubonja' + +huggingface_model_path = "https://huggingface.co/Ariel4/biobert-embeddings/resolve/main/pytorch_model.bin" #Create and configure logger logging.basicConfig(filename='app.log', filemode='w',format='%(asctime)s %(message)s', level=logging.INFO) @@ -26,10 +28,10 @@ class BiobertEmbedding(object): def __init__(self, model_path=None): - if model_path is not None: - self.model_path = model_path - else: - self.model_path = downloader.get_BioBert("google drive") + if model_path is None: # If model doesn't exist locally, download + model_path = self.download_model(huggingface_model_path) + + self.model_path = model_path self.tokens = "" self.sentence_tokens = "" @@ -37,6 +39,28 @@ def __init__(self, model_path=None): # Load pre-trained model (weights) self.model = BertModel.from_pretrained(self.model_path) logger.info("Initialization Done !!") + + + def download_model(url): + response = requests.get(url, stream=True) + + if not os.path.exists("models"): + os.makedirs("models") + + filename = os.path.basename(url) + + print("Downloading BioBert model from HuggingFace") + + total = int(response.headers.get('content-length', 0)) + with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: + with open(filename, 'wb') as f: + for data in response.iter_content(chunk_size=1024): + size = f.write(data) + bar.update(size) + + print("Model Downloaded! It is stored in: models/"+filename) + + def process_text(self, text): diff --git a/setup.py b/setup.py index 30785d7..c4715ee 100644 --- a/setup.py +++ b/setup.py @@ -6,18 +6,17 @@ setuptools.setup( name="biobert-embedding", packages=['biobert_embedding'], - version="0.1.2", + version="0.1.3", author="Jitendra Jangid, Ariel Lubonja", author_email="jitujangid38@gmail.com, ariellubonja@live.com", description="Embeddings from BioBERT", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/overfitter/biobert_embedding", - download_url="https://github.com/overfitter/biobert_embedding/archive/v0.1.2.tar.gz", + url="https://github.com/ariellubonja/biobert_embedding", + download_url="https://github.com/ariellubonja/biobert_embedding/archive/v0.1.3.tar.gz", install_requires=[ 'torch==1.13.1', 'pytorch-pretrained-bert==0.6.2', - 'tensorflow', ], classifiers=[ From 12d96608cf697544c0009e7bcc0bf92f0cc15f47 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Sat, 1 Jul 2023 22:36:45 -0400 Subject: [PATCH 04/11] Fix dumb function calling mistake. Also remove Tensorflow from requirements --- biobert_embedding/embedding.py | 44 ++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/biobert_embedding/embedding.py b/biobert_embedding/embedding.py index 1f664ba..ea521a0 100644 --- a/biobert_embedding/embedding.py +++ b/biobert_embedding/embedding.py @@ -15,6 +15,28 @@ logger = logging.getLogger(__name__) + +def download_model(url): + response = requests.get(url, stream=True) + + if not os.path.exists("models"): + os.makedirs("models") + + filename = os.path.basename(url) + + print("Downloading BioBert model from HuggingFace") + + total = int(response.headers.get('content-length', 0)) + with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: + with open(filename, 'wb') as f: + for data in response.iter_content(chunk_size=1024): + size = f.write(data) + bar.update(size) + + print("Model Downloaded! It is stored in: models/"+filename) + + return "models/"+filename + class BiobertEmbedding(object): """ Encoding from BioBERT model (BERT finetuned on PubMed articles). @@ -29,7 +51,7 @@ class BiobertEmbedding(object): def __init__(self, model_path=None): if model_path is None: # If model doesn't exist locally, download - model_path = self.download_model(huggingface_model_path) + model_path = download_model(huggingface_model_path) self.model_path = model_path @@ -39,26 +61,6 @@ def __init__(self, model_path=None): # Load pre-trained model (weights) self.model = BertModel.from_pretrained(self.model_path) logger.info("Initialization Done !!") - - - def download_model(url): - response = requests.get(url, stream=True) - - if not os.path.exists("models"): - os.makedirs("models") - - filename = os.path.basename(url) - - print("Downloading BioBert model from HuggingFace") - - total = int(response.headers.get('content-length', 0)) - with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: - with open(filename, 'wb') as f: - for data in response.iter_content(chunk_size=1024): - size = f.write(data) - bar.update(size) - - print("Model Downloaded! It is stored in: models/"+filename) From 35e592ce993046723baa0508a11123258a1b4616 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Mon, 3 Jul 2023 13:05:00 -0400 Subject: [PATCH 05/11] Fix model download to path --- biobert_embedding/embedding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/biobert_embedding/embedding.py b/biobert_embedding/embedding.py index ea521a0..a9ce05c 100644 --- a/biobert_embedding/embedding.py +++ b/biobert_embedding/embedding.py @@ -28,7 +28,7 @@ def download_model(url): total = int(response.headers.get('content-length', 0)) with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: - with open(filename, 'wb') as f: + with open("models/" + filename, 'wb') as f: for data in response.iter_content(chunk_size=1024): size = f.write(data) bar.update(size) From 7dcf74c7d39fa9a9331f9eab2f0bf873d11cde46 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Mon, 3 Jul 2023 13:08:47 -0400 Subject: [PATCH 06/11] Fix some logic --- biobert_embedding/embedding.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/biobert_embedding/embedding.py b/biobert_embedding/embedding.py index a9ce05c..3b0de95 100644 --- a/biobert_embedding/embedding.py +++ b/biobert_embedding/embedding.py @@ -17,23 +17,26 @@ def download_model(url): - response = requests.get(url, stream=True) + filename = os.path.basename(url) - if not os.path.exists("models"): - os.makedirs("models") + if os.path.isfile("models/" + filename): + print(f"Using existing models/" + filename) + else: + response = requests.get(url, stream=True) - filename = os.path.basename(url) + if not os.path.exists("models"): + os.makedirs("models") - print("Downloading BioBert model from HuggingFace") + print("Downloading BioBert model from HuggingFace") - total = int(response.headers.get('content-length', 0)) - with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: - with open("models/" + filename, 'wb') as f: - for data in response.iter_content(chunk_size=1024): - size = f.write(data) - bar.update(size) + total = int(response.headers.get('content-length', 0)) + with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: + with open("models/" + filename, 'wb') as f: + for data in response.iter_content(chunk_size=1024): + size = f.write(data) + bar.update(size) - print("Model Downloaded! It is stored in: models/"+filename) + print("Model Downloaded! It is stored in: models/"+filename) return "models/"+filename From 557b4dfe8d00ed86165adcafd8e8e819ba184bef Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Mon, 3 Jul 2023 13:52:28 -0400 Subject: [PATCH 07/11] Loading model requires path to folder containing the 3 files, not path to the pytorch file. Fixed --- biobert_embedding/embedding.py | 50 +++++++++++++++++++++------------- setup.py | 2 +- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/biobert_embedding/embedding.py b/biobert_embedding/embedding.py index 3b0de95..953f57e 100644 --- a/biobert_embedding/embedding.py +++ b/biobert_embedding/embedding.py @@ -3,12 +3,13 @@ import logging import requests from tqdm import tqdm -from urllib.parse import urlparse -from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM +from pytorch_pretrained_bert import BertTokenizer, BertModel __author__ = 'Jitendra Jangid, Ariel Lubonja' -huggingface_model_path = "https://huggingface.co/Ariel4/biobert-embeddings/resolve/main/pytorch_model.bin" + +huggingface_repo = "https://huggingface.co/Ariel4/biobert-embeddings/resolve/main/" + #Create and configure logger logging.basicConfig(filename='app.log', filemode='w',format='%(asctime)s %(message)s', level=logging.INFO) @@ -16,29 +17,41 @@ logger = logging.getLogger(__name__) -def download_model(url): - filename = os.path.basename(url) - - if os.path.isfile("models/" + filename): - print(f"Using existing models/" + filename) +def download_or_use_existing(model_folder_path, filename): + if os.path.isfile(model_folder_path + filename): + print(f"Using existing " + model_folder_path + filename) else: - response = requests.get(url, stream=True) + # Download with Progress Bar + response = requests.get(huggingface_repo + filename, stream=True) - if not os.path.exists("models"): - os.makedirs("models") - - print("Downloading BioBert model from HuggingFace") + print("Downloading " + filename + " from HuggingFace") total = int(response.headers.get('content-length', 0)) with tqdm(total=total, unit='iB', unit_scale=True, ncols=70) as bar: - with open("models/" + filename, 'wb') as f: + with open(model_folder_path + filename, 'wb') as f: for data in response.iter_content(chunk_size=1024): size = f.write(data) bar.update(size) - print("Model Downloaded! It is stored in: models/"+filename) + print("File Downloaded! It is stored in: " + model_folder_path+filename) - return "models/"+filename + +def setup_model(model_folder_path="models/"): + """ + Verify if the model is already downloaded, if not download it. + """ + pytorch_model_filename = "pytorch_model.bin" + config_json_filename = "config.json" + vocab_filename = "vocab.txt" + + if not os.path.exists(model_folder_path): + os.makedirs(model_folder_path) + + download_or_use_existing(model_folder_path, pytorch_model_filename) + download_or_use_existing(model_folder_path, config_json_filename) + download_or_use_existing(model_folder_path, vocab_filename) + + return model_folder_path class BiobertEmbedding(object): """ @@ -52,14 +65,13 @@ class BiobertEmbedding(object): """ def __init__(self, model_path=None): - - if model_path is None: # If model doesn't exist locally, download - model_path = download_model(huggingface_model_path) + model_path = setup_model() # Folder containing pytorch_model.bin, config.json and vocab.txt self.model_path = model_path self.tokens = "" self.sentence_tokens = "" + # This needs the model folder path, not the pytorch_model.bin path self.tokenizer = BertTokenizer.from_pretrained(self.model_path) # Load pre-trained model (weights) self.model = BertModel.from_pretrained(self.model_path) diff --git a/setup.py b/setup.py index c4715ee..a1f2b49 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="biobert-embedding", packages=['biobert_embedding'], - version="0.1.3", + version="0.1.4", author="Jitendra Jangid, Ariel Lubonja", author_email="jitujangid38@gmail.com, ariellubonja@live.com", description="Embeddings from BioBERT", From 189459f6dfb4aef0479c4b0ccb2ebcfbf90fd57d Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Mon, 3 Jul 2023 14:20:40 -0400 Subject: [PATCH 08/11] Fixed undefined distance() in README example code --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a3a6ffc..7910e24 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ sentence embedding generated is 768 dimensional embedding. ```python from biobert_embedding.embedding import BiobertEmbedding +from scipy.spatial.distance import cosine as cosine_distance ## Example 1 text = "Breast cancers with HER2 amplification have a higher risk of CNS metastasis and poorer prognosis."\ @@ -52,7 +53,7 @@ print("Shape of Sentence Embedding = ",len(sentence_embedding)) sentence_vector1 = biobert.sentence_vector('Breast cancers with HER2 amplification have a higher risk of CNS metastasis and poorer prognosis.') sentence_vector2 = biobert.sentence_vector('Breast cancers with HER2 amplification are more aggressive, have a higher risk of CNS metastasis, and poorer prognosis.') -cosine_sim = 1 - distance.cosine(sentence_vector1, sentence_vector2) +cosine_sim = 1 - cosine_distance(sentence_vector1, sentence_vector2) print('cosine similarity:', cosine_sim) #cosine similarity: 0.992756187915802 ``` From 3d00fe3c8176b116af7f11f421a15d8a0c80170f Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Tue, 4 Jul 2023 20:44:58 -0400 Subject: [PATCH 09/11] Add comments --- biobert_embedding/embedding.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/biobert_embedding/embedding.py b/biobert_embedding/embedding.py index 953f57e..d2233f2 100644 --- a/biobert_embedding/embedding.py +++ b/biobert_embedding/embedding.py @@ -88,6 +88,9 @@ def process_text(self, text): def handle_oov(self, tokenized_text, word_embeddings): + """ + Handle out-of-vocabulary words by appending the word embeddings of the subwords + """ embeddings = [] tokens = [] oov_len = 1 From deeacb6cfce6b5fe3222251f8ed7ad14545f2bc4 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Mon, 26 Feb 2024 16:59:31 -0500 Subject: [PATCH 10/11] Upgrade torch version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a1f2b49..cc75ff1 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ url="https://github.com/ariellubonja/biobert_embedding", download_url="https://github.com/ariellubonja/biobert_embedding/archive/v0.1.3.tar.gz", install_requires=[ - 'torch==1.13.1', + 'torch==2.1.2', 'pytorch-pretrained-bert==0.6.2', ], From 1313ea71f2dec3ee42cc89679fa2b0a247d99891 Mon Sep 17 00:00:00 2001 From: Ariel Lubonja Date: Wed, 19 Jun 2024 12:59:44 -0400 Subject: [PATCH 11/11] Add .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af56f61 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +.idea/