diff options
Diffstat (limited to '')
| -rw-r--r-- | cogs/utils/file_search.py (renamed from cogs/music/search_utils.py) | 279 |
1 files changed, 246 insertions, 33 deletions
diff --git a/cogs/music/search_utils.py b/cogs/utils/file_search.py index bc9e335..4a2bdc4 100644 --- a/cogs/music/search_utils.py +++ b/cogs/utils/file_search.py | |||
| @@ -3,12 +3,57 @@ | |||
| 3 | # Copyright (C) 2026 Salmonik @ salmonik@tuta.io | 3 | # Copyright (C) 2026 Salmonik @ salmonik@tuta.io |
| 4 | # https://codeberg.org/salmonik/senator-sperma | 4 | # https://codeberg.org/salmonik/senator-sperma |
| 5 | 5 | ||
| 6 | """ | ||
| 7 | Universal File Searcher | ||
| 8 | |||
| 9 | This module provides a universal file searcher that can search for different file types: | ||
| 10 | - Audio files (mp3, wav, mp4, opus, m4a, flac, aac, ogg) | ||
| 11 | - Video files (mp4, avi, mkv, mov, wmv, flv, webm, m4v, 3gp) | ||
| 12 | - Image files (jpg, jpeg, png, gif, bmp, tiff, webp, svg, ico) | ||
| 13 | |||
| 14 | Usage Examples: | ||
| 15 | # Search all file types | ||
| 16 | results = await UniversalSearcher.search_local("query") | ||
| 17 | |||
| 18 | # Search only audio files | ||
| 19 | results = await UniversalSearcher.search_local("query", filetype="Audio") | ||
| 20 | |||
| 21 | # Search only video files | ||
| 22 | results = await UniversalSearcher.search_local("query", filetype="Video") | ||
| 23 | |||
| 24 | # Search only image files | ||
| 25 | results = await UniversalSearcher.search_local("query", filetype="Image") | ||
| 26 | |||
| 27 | # Autocomplete search | ||
| 28 | results = await UniversalSearcher.search_local_autocomplete("partial_query", filetype="Video") | ||
| 29 | |||
| 30 | Settings.json structure: | ||
| 31 | { | ||
| 32 | "file_paths": [ | ||
| 33 | "/path/to/your/media/files", | ||
| 34 | "/another/path/with/mixed/content", | ||
| 35 | "relative/path/to/files" | ||
| 36 | ] | ||
| 37 | } | ||
| 38 | |||
| 39 | Note: file_paths should contain directories that may contain any mix of audio, video, | ||
| 40 | and image files. The searcher will filter by file type automatically based on the | ||
| 41 | filetype parameter. | ||
| 42 | """ | ||
| 43 | |||
| 6 | import json | 44 | import json |
| 7 | import logging | 45 | import logging |
| 8 | import re | 46 | import re |
| 47 | import urllib.parse | ||
| 9 | from pathlib import Path | 48 | from pathlib import Path |
| 10 | import aiofiles | 49 | import aiofiles |
| 11 | from rapidfuzz import fuzz | 50 | from rapidfuzz import fuzz |
| 51 | import aiohttp | ||
| 52 | import asyncio | ||
| 53 | import os | ||
| 54 | from dotenv import load_dotenv | ||
| 55 | |||
| 56 | load_dotenv() | ||
| 12 | 57 | ||
| 13 | logger = logging.getLogger(__name__) | 58 | logger = logging.getLogger(__name__) |
| 14 | 59 | ||
| @@ -173,14 +218,34 @@ class SearchUtils: | |||
| 173 | 218 | ||
| 174 | return score | 219 | return score |
| 175 | 220 | ||
| 176 | class AudioSearcher: | 221 | class UniversalSearcher: |
| 177 | """Handles searching for audio files in configured paths.""" | 222 | """Handles searching for files in configured paths with support for multiple file types.""" |
| 223 | |||
| 224 | FILE_TYPE_EXTENSIONS = { | ||
| 225 | 'Audio': {'.mp4', '.mp3', '.wav', '.opus', '.m4a', '.flac', '.aac', '.ogg'}, | ||
| 226 | 'Video': {'.mp4', '.avi', '.mkv', '.mov', '.wmv', '.flv', '.webm', '.m4v', '.3gp', '.gif'}, | ||
| 227 | 'Image': {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp', '.svg', '.ico'} | ||
| 228 | } | ||
| 178 | 229 | ||
| 179 | @staticmethod | 230 | @staticmethod |
| 180 | def _is_valid_audio_file(file_path: Path) -> bool: | 231 | def _is_valid_file(file_path: Path, filetype: str = None) -> bool: |
| 181 | """Check if file is a valid audio file.""" | 232 | """Check if file is valid for the specified filetype.""" |
| 182 | valid_extensions = {'.mp4', '.mp3', '.wav', '.opus', '.m4a'} | 233 | if not file_path.is_file(): |
| 183 | return file_path.is_file() and file_path.suffix.lower() in valid_extensions | 234 | return False |
| 235 | |||
| 236 | if filetype is None: | ||
| 237 | # Search all file types | ||
| 238 | all_extensions = set() | ||
| 239 | for extensions in UniversalSearcher.FILE_TYPE_EXTENSIONS.values(): | ||
| 240 | all_extensions.update(extensions) | ||
| 241 | return file_path.suffix.lower() in all_extensions | ||
| 242 | |||
| 243 | filetype = filetype.capitalize() | ||
| 244 | if filetype not in UniversalSearcher.FILE_TYPE_EXTENSIONS: | ||
| 245 | return False | ||
| 246 | |||
| 247 | valid_extensions = UniversalSearcher.FILE_TYPE_EXTENSIONS[filetype] | ||
| 248 | return file_path.suffix.lower() in valid_extensions | ||
| 184 | 249 | ||
| 185 | @staticmethod | 250 | @staticmethod |
| 186 | def _calculate_match_score(query: str, file_name: str, autocomplete: bool) -> tuple[int, str]: | 251 | def _calculate_match_score(query: str, file_name: str, autocomplete: bool) -> tuple[int, str]: |
| @@ -201,18 +266,18 @@ class AudioSearcher: | |||
| 201 | return -1, file_name | 266 | return -1, file_name |
| 202 | 267 | ||
| 203 | @staticmethod | 268 | @staticmethod |
| 204 | async def _search_single_path(path: Path, query: str, max_results: int, autocomplete: bool) -> list[tuple[int, str]]: | 269 | async def _search_single_path(path: Path, query: str, max_results: int, autocomplete: bool, filetype: str = None) -> list[tuple[int, str]]: |
| 205 | """Search for audio files in a single path.""" | 270 | """Search for files in a single path.""" |
| 206 | matches = [] | 271 | matches = [] |
| 207 | 272 | ||
| 208 | logger.debug(f"Searching in directory: {path}") | 273 | logger.debug(f"Searching in directory: {path} (filetype: {filetype or 'All'})") |
| 209 | 274 | ||
| 210 | for file_path in path.rglob('*'): | 275 | for file_path in path.rglob('*'): |
| 211 | if not AudioSearcher._is_valid_audio_file(file_path): | 276 | if not UniversalSearcher._is_valid_file(file_path, filetype): |
| 212 | continue | 277 | continue |
| 213 | 278 | ||
| 214 | file_name = file_path.name.lower() | 279 | file_name = file_path.name.lower() |
| 215 | score, _ = AudioSearcher._calculate_match_score(query, file_name, autocomplete) | 280 | score, _ = UniversalSearcher._calculate_match_score(query, file_name, autocomplete) |
| 216 | 281 | ||
| 217 | if score >= 0: | 282 | if score >= 0: |
| 218 | matches.append((score, str(file_path.absolute()))) | 283 | matches.append((score, str(file_path.absolute()))) |
| @@ -222,14 +287,41 @@ class AudioSearcher: | |||
| 222 | 287 | ||
| 223 | return matches | 288 | return matches |
| 224 | 289 | ||
| 290 | @staticmethod | ||
| 291 | def _get_search_paths(data: dict, filetype: str = None) -> list: | ||
| 292 | """Get the search paths from settings.""" | ||
| 293 | return data.get('file_paths', []) | ||
| 294 | |||
| 225 | @classmethod | 295 | @classmethod |
| 226 | async def search_local(cls, query: str = None, max_results: int = 25, autocomplete: bool = False): | 296 | async def _search_paths(cls, search_paths: list, query: str, max_results: int, autocomplete: bool, filetype: str = None) -> tuple[list, int]: |
| 227 | """Search for audio files with fuzzy matching and optional autocomplete support. | 297 | """Search through the provided paths and return matches and file count.""" |
| 298 | matches = [] | ||
| 299 | file_count = 0 | ||
| 300 | |||
| 301 | for path_str in search_paths: | ||
| 302 | path = Path(path_str) | ||
| 303 | if not path.exists(): | ||
| 304 | logger.warning(f"Path does not exist: {path}") | ||
| 305 | continue | ||
| 306 | |||
| 307 | path_matches = await cls._search_single_path(path, query, max_results, autocomplete, filetype) | ||
| 308 | matches.extend(path_matches) | ||
| 309 | file_count += len(path_matches) | ||
| 310 | |||
| 311 | if len(matches) >= max_results * 5: | ||
| 312 | break | ||
| 313 | |||
| 314 | return matches, file_count | ||
| 315 | |||
| 316 | @classmethod | ||
| 317 | async def search_local(cls, query: str = None, max_results: int = 25, autocomplete: bool = False, filetype: str = None): | ||
| 318 | """Search for files with fuzzy matching and optional autocomplete support. | ||
| 228 | 319 | ||
| 229 | Args: | 320 | Args: |
| 230 | query: Search query string (can be None for autocomplete) | 321 | query: Search query string (can be None for autocomplete) |
| 231 | max_results: Maximum number of results to return | 322 | max_results: Maximum number of results to return |
| 232 | autocomplete: If True, returns results even with empty query | 323 | autocomplete: If True, returns results even with empty query |
| 324 | filetype: Type of files to search for ('Audio', 'Video', 'Image', or None for all) | ||
| 233 | 325 | ||
| 234 | Returns: | 326 | Returns: |
| 235 | List of matching file paths with scores | 327 | List of matching file paths with scores |
| @@ -238,30 +330,17 @@ class AudioSearcher: | |||
| 238 | return [] | 330 | return [] |
| 239 | 331 | ||
| 240 | query = (query or '').lower().strip() | 332 | query = (query or '').lower().strip() |
| 241 | logger.info(f"Starting local search for query: '{query}'") | 333 | logger.info(f"Starting local search for query: '{query}' (filetype: {filetype or 'All'})") |
| 242 | 334 | ||
| 243 | try: | 335 | try: |
| 244 | async with aiofiles.open('settings.json') as f: | 336 | async with aiofiles.open('settings.json') as f: |
| 245 | content = await f.read() | 337 | content = await f.read() |
| 246 | data = json.loads(content) | 338 | data = json.loads(content) |
| 247 | search_paths = data.get('audio_paths', []) | ||
| 248 | logger.info(f"Searching in paths: {search_paths}") | ||
| 249 | 339 | ||
| 250 | matches = [] | 340 | search_paths = cls._get_search_paths(data, filetype) |
| 251 | file_count = 0 | 341 | logger.info(f"Searching in paths: {search_paths}") |
| 252 | 342 | ||
| 253 | for path_str in search_paths: | 343 | matches, file_count = await cls._search_paths(search_paths, query, max_results, autocomplete, filetype) |
| 254 | path = Path(path_str) | ||
| 255 | if not path.exists(): | ||
| 256 | logger.warning(f"Path does not exist: {path}") | ||
| 257 | continue | ||
| 258 | |||
| 259 | path_matches = await cls._search_single_path(path, query, max_results, autocomplete) | ||
| 260 | matches.extend(path_matches) | ||
| 261 | file_count += len(path_matches) | ||
| 262 | |||
| 263 | if len(matches) >= max_results * 5: | ||
| 264 | break | ||
| 265 | 344 | ||
| 266 | logger.info(f"Scanned {file_count} files total") | 345 | logger.info(f"Scanned {file_count} files total") |
| 267 | 346 | ||
| @@ -278,10 +357,144 @@ class AudioSearcher: | |||
| 278 | return [] | 357 | return [] |
| 279 | 358 | ||
| 280 | @classmethod | 359 | @classmethod |
| 281 | async def search_local_autocomplete(cls, query: str = None, max_results: int = 10): | 360 | async def search_local_autocomplete(cls, query: str = None, max_results: int = 10, filetype: str = None): |
| 282 | """Wrapper for autocomplete searches that returns more results and allows empty query.""" | 361 | """Wrapper for autocomplete searches that returns more results and allows empty query. |
| 362 | |||
| 363 | Args: | ||
| 364 | query: Search query string (can be None for autocomplete) | ||
| 365 | max_results: Maximum number of results to return | ||
| 366 | filetype: Type of files to search for ('Audio', 'Video', 'Image', or None for all) | ||
| 367 | """ | ||
| 283 | # For autocomplete, we want to show more options and allow empty queries | 368 | # For autocomplete, we want to show more options and allow empty queries |
| 284 | if not query: | 369 | if not query: |
| 285 | return [] | 370 | return [] |
| 286 | 371 | ||
| 287 | return await cls.search_local(query, max_results=max_results, autocomplete=True) | 372 | return await cls.search_local(query, max_results=max_results, autocomplete=True, filetype=filetype) |
| 373 | |||
| 374 | @staticmethod | ||
| 375 | async def _load_baseurl(): | ||
| 376 | """Load baseurl from settings.json""" | ||
| 377 | try: | ||
| 378 | async with aiofiles.open('settings.json') as f: | ||
| 379 | content = await f.read() | ||
| 380 | data = json.loads(content) | ||
| 381 | return data.get('baseurl', 'http://localhost:8080').rstrip('/') | ||
| 382 | except Exception as e: | ||
| 383 | logger.error(f"Error loading baseurl: {e}") | ||
| 384 | return 'http://localhost:8080' | ||
| 385 | |||
| 386 | @staticmethod | ||
| 387 | async def search_copyparty(query: str, max_results: int = 25): | ||
| 388 | """Search for files using copyparty API (jPOST method). | ||
| 389 | |||
| 390 | Args: | ||
| 391 | query: Search query string | ||
| 392 | max_results: Maximum number of results to return | ||
| 393 | |||
| 394 | Returns: | ||
| 395 | List of matching file URLs | ||
| 396 | """ | ||
| 397 | |||
| 398 | |||
| 399 | if not query: | ||
| 400 | return [] | ||
| 401 | |||
| 402 | query = query.strip() | ||
| 403 | logger.info(f"Starting copyparty search for query: '{query}'") | ||
| 404 | query = f'name like "*{query}*"' | ||
| 405 | |||
| 406 | try: | ||
| 407 | baseurl = await UniversalSearcher._load_baseurl() | ||
| 408 | password = os.getenv("COPYPARTY_R_PASSWORD") | ||
| 409 | |||
| 410 | async with aiohttp.ClientSession() as session: | ||
| 411 | headers = {"Content-Type": "application/json"} | ||
| 412 | if password: | ||
| 413 | headers["PW"] = password | ||
| 414 | async with session.post(f"{baseurl}/?srch", headers=headers, data=json.dumps({"n": max_results, "q": query})) as response: | ||
| 415 | if response.status == 200: | ||
| 416 | data = await response.json() | ||
| 417 | logger.info(f"Copyparty response: {data}") | ||
| 418 | files = UniversalSearcher._extract_files_from_response(data) | ||
| 419 | results = UniversalSearcher._convert_files_to_urls(files, baseurl, max_results) | ||
| 420 | |||
| 421 | logger.info(f"Copyparty search complete. Found {len(results)} matches.") | ||
| 422 | return results | ||
| 423 | else: | ||
| 424 | logger.error(f"Copyparty API returned status {response.status}") | ||
| 425 | return [] | ||
| 426 | |||
| 427 | except aiohttp.ClientError as e: | ||
| 428 | logger.error(f"HTTP error during copyparty search: {e}") | ||
| 429 | return [] | ||
| 430 | except Exception as e: | ||
| 431 | logger.error(f"Error during copyparty search: {str(e)}", exc_info=True) | ||
| 432 | return [] | ||
| 433 | |||
| 434 | @staticmethod | ||
| 435 | def _extract_files_from_response(data): | ||
| 436 | """Extract files list from copyparty API response.""" | ||
| 437 | if isinstance(data, dict): | ||
| 438 | if 'hits' in data: | ||
| 439 | return data['hits'] | ||
| 440 | elif 'files' in data: | ||
| 441 | return data['files'] | ||
| 442 | elif 'results' in data: | ||
| 443 | return data['results'] | ||
| 444 | else: | ||
| 445 | return data if isinstance(data, list) else [] | ||
| 446 | elif isinstance(data, list): | ||
| 447 | return data | ||
| 448 | else: | ||
| 449 | return [] | ||
| 450 | |||
| 451 | @staticmethod | ||
| 452 | def _convert_files_to_urls(files, baseurl, max_results): | ||
| 453 | """Convert file info objects to URLs.""" | ||
| 454 | results = [] | ||
| 455 | |||
| 456 | for file_info in files[:max_results]: | ||
| 457 | if isinstance(file_info, dict): | ||
| 458 | file_url = UniversalSearcher._extract_url_from_dict(file_info, baseurl) | ||
| 459 | if file_url: | ||
| 460 | results.append(file_url) | ||
| 461 | elif isinstance(file_info, str): | ||
| 462 | # If it's just a string path/URL | ||
| 463 | if file_info.startswith('http'): | ||
| 464 | results.append(file_info) | ||
| 465 | else: | ||
| 466 | results.append(f"{baseurl}/{file_info.lstrip('/')}") | ||
| 467 | |||
| 468 | return results | ||
| 469 | |||
| 470 | @staticmethod | ||
| 471 | def _extract_url_from_dict(file_info, baseurl): | ||
| 472 | """Extract URL from file info dictionary.""" | ||
| 473 | if 'url' in file_info: | ||
| 474 | return file_info['url'] | ||
| 475 | elif 'rp' in file_info: | ||
| 476 | return f"{baseurl}/{file_info['rp']}" | ||
| 477 | elif 'path' in file_info: | ||
| 478 | return f"{baseurl}/{file_info['path'].lstrip('/')}" | ||
| 479 | elif 'name' in file_info: | ||
| 480 | return f"{baseurl}/{file_info['name'].lstrip('/')}" | ||
| 481 | else: | ||
| 482 | return None | ||
| 483 | |||
| 484 | @staticmethod | ||
| 485 | async def search_copyparty_autocomplete(query: str, max_results: int = 10): | ||
| 486 | """Wrapper for autocomplete searches using copyparty. | ||
| 487 | |||
| 488 | Args: | ||
| 489 | query: Search query string | ||
| 490 | max_results: Maximum number of results to return | ||
| 491 | |||
| 492 | Returns: | ||
| 493 | List of matching file URLs | ||
| 494 | """ | ||
| 495 | if not query: | ||
| 496 | return [] | ||
| 497 | |||
| 498 | return await UniversalSearcher.search_copyparty(query, max_results) | ||
| 499 | |||
| 500 | |||
