Day 48: NLP: Text Cleaning & Sentiment Model

import re
import pandas as pd
r = pd.read_csv("product_reviews.csv")
print(r["sentiment"].value_counts().to_dict())
def clean(t):
    t = t.lower()
    t = re.sub(r"[^a-z\s]", " ", t)          # keep letters only
    return re.sub(r"\s+", " ", t).strip()
r["clean"] = r["review"].apply(clean)
print(r[["review", "clean"]].head(4).to_string(index=False))

from sklearn.feature_extraction.text import CountVectorizer
docs = ["battery life is amazing", "battery died fast, very bad", "amazing sound, amazing price"]
cv = CountVectorizer()
M = cv.fit_transform(docs)
import pandas as pd
print(pd.DataFrame(M.toarray(), columns=cv.get_feature_names_out(), index=["r1", "r2", "r3"]))

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
r = pd.read_csv("product_reviews.csv")
tf = TfidfVectorizer(stop_words="english", ngram_range=(1, 2), min_df=2)
X = tf.fit_transform(r["review"])
print("matrix:", X.shape, "| sparsity:", round(1 - X.nnz / (X.shape[0] * X.shape[1]), 3))
row = pd.Series(X[0].toarray().ravel(), index=tf.get_feature_names_out())
print("review 0:", r.loc[0, "review"]); print(row[row > 0].sort_values(ascending=False).round(3))

import pandas as pd
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.pipeline import make_pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
r = pd.read_csv("product_reviews.csv")
X_tr, X_te, y_tr, y_te = train_test_split(r["review"], r["sentiment"], test_size=0.25, random_state=0, stratify=r["sentiment"])
for name, clf in [("Naive Bayes", MultinomialNB()), ("Logistic regression", LogisticRegression(max_iter=2000))]:
    pipe = make_pipeline(TfidfVectorizer(ngram_range=(1, 2), min_df=2), clf)
    print(f"{name:20s} 5-fold CV accuracy = {cross_val_score(pipe, X_tr, y_tr, cv=5).mean():.3f}")
best = make_pipeline(TfidfVectorizer(ngram_range=(1, 2), min_df=2), LogisticRegression(max_iter=2000)).fit(X_tr, y_tr)
print(classification_report(y_te, best.predict(X_te), digits=3))

import pandas as pd
import matplotlib.pyplot as plt
from sklearn.pipeline import make_pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
r = pd.read_csv("product_reviews.csv")
pipe = make_pipeline(TfidfVectorizer(stop_words="english", min_df=2), LogisticRegression(max_iter=2000)).fit(r["review"], r["sentiment"])
coef = pd.Series(pipe[-1].coef_[0], index=pipe[0].get_feature_names_out())      # positive class = "positive"
top = pd.concat([coef.nsmallest(10), coef.nlargest(10)])
plt.figure(figsize=(8, 5.2))
plt.barh(top.index, top.values, color=["#dc2626" if v < 0 else "#16a34a" for v in top.values])
plt.title("Words that push a review negative (red) or positive (green)"); plt.xlabel("coefficient")
plt.show()

import pandas as pd
from sklearn.pipeline import make_pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
r = pd.read_csv("product_reviews.csv")
pipe = make_pipeline(TfidfVectorizer(ngram_range=(1, 2), min_df=2), LogisticRegression(max_iter=2000)).fit(r["review"], r["sentiment"])
tests = ["Battery life is amazing, loved it", "Stopped working in a week, waste of money",
         "Not bad at all, quite good actually", "delivery late but product is superb",
         "bilkul bekaar product, paisa barbaad", "Phone is ok"]
for t, p in zip(tests, pipe.predict_proba(tests)[:, list(pipe.classes_).index("positive")]):
    print(f"{p:5.2f}  {'POS' if p >= 0.5 else 'NEG'}  {t}")

import pandas as pd
r = pd.read_csv("product_reviews.csv")
neg = r[r["sentiment"] == "negative"]["review"].str.lower()
topics = {"battery / charging": "battery|charg", "quality / broken": "broke|stopped|crack|faded|quality",
          "size / fit": "size|fit|tight|small", "fake / value": "fake|waste|money|original",
          "delivery": "deliver|late|courier"}
counts = {k: int(neg.str.contains(v).sum()) for k, v in topics.items()}
s = pd.Series(counts).sort_values(ascending=False)
for topic, n in s.items():
    print(f"{topic:20s} {n:3d} reviews  ({n / len(neg) * 100:.1f}% of negative)")