Compare commits

..

No commits in common. "master" and "v0.2.1" have entirely different histories.

83 changed files with 47 additions and 171 deletions

23
.github/workflows/publish.yml vendored Normal file
View file

@ -0,0 +1,23 @@
name: Publish to PyPI
on:
release:
types: [published]
jobs:
publish:
runs-on: ubuntu-latest
environment: pypi
permissions:
id-token: write
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
- name: Build
run: uv build
- name: Publish
uses: pypa/gh-action-pypi-publish@release/v1

View file

@ -1,34 +0,0 @@
name: Release
on:
workflow_dispatch:
jobs:
release:
runs-on: ubuntu-latest
environment: pypi
permissions:
id-token: write
contents: write
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
- name: Get version from pyproject.toml
id: get_version
run: |
VERSION=$(python -c "import tomllib; data=tomllib.load(open('pyproject.toml','rb')); print(data['project']['version'])")
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Build
run: uv build
- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
tag_name: v${{ steps.get_version.outputs.version }}
name: v${{ steps.get_version.outputs.version }}
generate_release_notes: true
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1

View file

@ -20,10 +20,8 @@ w.get_profane_words("نص فيه حرامي و أطرش") # ['حرامي', 'أ
```
> [!NOTE]
> تدعم المكتبة إزالة التشكيل تلقائياً عند استخدام اللغة العربية
> المكتبة تدعم إزالة التشكيل من الكلمات تلقائياً
> [!TIP]
> يدعم المشروع النمط البديل (Wildcard) في قوائم الكلمات — استخدم `*` للتطابق مع أي تسلسل من الأحرف (مثال: `bad*` تطابق `badly`، و`*word*` تطابق أي كلمة تحتوي على `word`)
## اللغات المدعومة

View file

@ -22,11 +22,6 @@ w.is_profane("Hello World") # False
w.get_profane_words("this is damn annoying") # ['damn']
```
> [!NOTE]
> The library automatically removes Arabic diacritics (Tashkeel) when using Arabic language mode
> [!TIP]
> Wildcard patterns are supported in word lists — use `*` to match any sequence of characters (e.g., `bad*` matches `badly`, `*word*` matches anything containing `word`)
## Supported Languages

View file

@ -966,7 +966,7 @@ zwimel
ابو فص
ابو قرعة
اتن
*احا*
احا
احترم نفسك
احتلام
احلي كث
@ -1275,11 +1275,10 @@ zwimel
نكت امه
نياكة
نياكه
*نيك*
نيك
واطي
وسخ
ولد القحبة
ولد القحبه
يا هبيلة
يلعن
*كس*

View file

@ -1,6 +1,6 @@
[project]
name = "wiqaya"
version = "0.2.5"
version = "0.2.0"
description = "A Python library for multilingual profanity detection and filtering. It identifies and censors offensive or abusive words across multiple languages."
readme = "README.md"
license = {text = "MIT"}
@ -18,6 +18,3 @@ build-backend = "uv_build"
dev = [
"pytest>=9.0.2",
]
[tool.setuptools.package-data]
wiqaya = ["data/*.txt"]

View file

@ -1,123 +1,38 @@
from pathlib import Path
from pathlib import Path
from .utils import remove_tashkeel
import re
DATA_DIR = Path(__file__).parent / "data"
DATA_DIR = Path(__file__).parent.parent.parent / "data"
class Wiqaya:
def __init__(self, lang: str):
"""
Initialize the Wiqaya profanity filter for a given language.
Loads the word list from a language-specific .txt file in the data directory.
Entries containing '*' are treated as wildcard patterns and compiled into
regex objects. Plain entries are stored in a set for O(1) lookup.
Args:
lang (str): Language code (e.g., 'ar', 'en'). Must match a filename in data/.
Raises:
ValueError: If no word list file exists for the given language.
"""
self.lang = lang
try:
with open(f"{DATA_DIR}/{self.lang}.txt", "r", encoding="utf-8") as f:
lines = [line.strip() for line in f if line.strip()]
self.WORDS = set(line.strip() for line in f)
except FileNotFoundError:
raise ValueError(f"Language '{self.lang}' not supported")
self.WORDS = set()
self._patterns = []
for entry in lines:
if "*" in entry:
# Convert wildcard to regex: *word* → .*word.*, word* → word.*
regex = re.escape(entry).replace(r"\*", ".*")
self._patterns.append(re.compile(f"^{regex}$"))
else:
self.WORDS.add(entry)
def is_profane(self, text) -> bool:
words = self._process(text)
return any(word in self.WORDS for word in words)
def _matches_any_pattern(self, word: str) -> bool:
"""
Check whether a word matches any of the compiled wildcard regex patterns.
Args:
word (str): The word to test.
Returns:
bool: True if the word matches at least one pattern, False otherwise.
"""
return any(p.match(word) for p in self._patterns)
def _is_bad(self, word: str) -> bool:
"""
Determine if a single word is considered profane.
Checks both the exact-match word set and the wildcard pattern list.
Args:
word (str): The word to check.
Returns:
bool: True if the word is profane, False otherwise.
"""
return word in self.WORDS or self._matches_any_pattern(word)
def is_profane(self, text: str) -> bool:
"""
Return True if the text contains at least one profane word.
Args:
text (str): The input text to scan.
Returns:
bool: True if any profane word is found, False otherwise.
"""
return any(self._is_bad(w) for w in self._process(text))
def get_profane_words(self, text: str) -> list[str]:
"""
Extract and return all profane words found in the text.
Args:
text (str): The input text to scan.
Returns:
list[str]: A list of every word in the text that is considered profane.
"""
return [w for w in self._process(text) if self._is_bad(w)]
def get_profane_words(self, text) -> list[str]:
words = self._process(text)
return [word for word in words if word in self.WORDS]
def censor(self, text: str, char: str = "*") -> str:
"""
Replace each profane word in the text with a repeated censor character.
The replacement preserves the original word's length (e.g., 'hell''****').
Args:
text (str): The input text to censor.
char (str): The character used for censoring. Defaults to '*'.
Returns:
str: The censored version of the input text.
"""
for word in self._process(text):
if self._is_bad(word):
words = self._process(text)
for word in words:
if word in self.WORDS:
text = text.replace(word, char * len(word))
return text
def _process(self, text: str) -> list[str]:
"""
Normalize and tokenize the input text into a list of words.
For Arabic text, diacritics (tashkeel) are stripped first to prevent
users from bypassing the filter by adding vowel marks to profane words.
The text is then lowercased and split on whitespace.
Args:
text (str): The raw input text.
Returns:
list[str]: A list of normalized, lowercase tokens.
"""
if self.lang == "ar":
text = remove_tashkeel(text)
return text.lower().split()
return text.lower().split()

View file

@ -55,21 +55,4 @@ def test_get_profane_words_en():
def test_invalid_lang():
import pytest
with pytest.raises(ValueError):
Wiqaya(lang="xx")
def test_wildcard_support():
w = Wiqaya(lang="en")
# is_profane
assert w.is_profane("wwsfuck") == True
assert w.is_profane("fuckwedf") == True
assert w.is_profane("wd+wfucked+") == True
# get_profane_words
assert w.get_profane_words("hello fsdfuckwwq clean") == ["fsdfuckwwq"]
# censor
assert w.censor("hello dsfuckw there") == "hello ******* there"
assert w.censor("dsfuckw ffdamn", char="#") == "####### ######"
Wiqaya(lang="xx")

2
uv.lock generated
View file

@ -65,7 +65,7 @@ wheels = [
[[package]]
name = "wiqaya"
version = "0.2.5"
version = "0.2.0"
source = { editable = "." }
[package.dev-dependencies]