asxshorts

 29class ShortsClient:
 30    """Client for fetching ASX short selling data with caching."""
 31
 32    def __init__(
 33        self,
 34        *,
 35        settings: ClientSettings | None = None,
 36        session: requests.Session | None = None,
 37        resolver: UrlResolver | None = None,
 38        **kwargs: Any,
 39    ) -> None:
 40        """Initialize the ShortsClient.
 41
 42        Args:
 43            settings: Client settings (will be created from kwargs if not provided)
 44            session: Custom requests session
 45            resolver: Custom URL resolver
 46            **kwargs: Settings passed to ClientSettings if settings not provided
 47        """
 48        # Initialize settings
 49        if settings is None:
 50            self.settings = ClientSettings(**kwargs)
 51        else:
 52            self.settings = settings
 53
 54        # Initialize components
 55        self.cache = CacheManager(self.settings.cache_dir)
 56        self.session = session or self._create_session()
 57        if resolver is not None:
 58            self.resolver = resolver
 59        else:
 60            # Prefer ASIC JSON-based resolution with DefaultResolver fallback
 61            self.resolver = CompositeResolver(self.settings.base_url, self.session)
 62
 63        logger.debug(
 64            f"Initialized ShortsClient with cache_dir={self.settings.cache_dir}, base_url={self.settings.base_url}"
 65        )
 66
 67    # Intentionally avoid configuring global logging in a library.
 68    # The CLI configures logging; library users can configure as desired.
 69
 70    def _create_session(self) -> requests.Session:
 71        """Create configured requests session with retry logic."""
 72        session = requests.Session()
 73
 74        # Set headers
 75        session.headers.update(
 76            {
 77                "User-Agent": self.settings.user_agent,
 78                "Accept": "text/csv,text/plain,*/*",
 79                "Accept-Encoding": "gzip, deflate",
 80            }
 81        )
 82
 83        # Configure retry strategy
 84        if self.settings.http_adapter_retries:
 85            retry_strategy = Retry(
 86                total=self.settings.retries,
 87                status_forcelist=[429, 500, 502, 503, 504],
 88                backoff_factor=self.settings.backoff,
 89                allowed_methods=["HEAD", "GET", "OPTIONS"],
 90            )
 91            adapter = HTTPAdapter(max_retries=retry_strategy)
 92        else:
 93            # Disable adapter-level retries to avoid double backoff when using
 94            # the client's explicit retry loop in _fetch_with_retry.
 95            adapter = HTTPAdapter(max_retries=0)
 96        session.mount("http://", adapter)
 97        session.mount("https://", adapter)
 98
 99        return session
100
101    def _fetch_with_retry(self, url: str) -> bytes:
102        """Fetch URL content with exponential backoff retry."""
103        last_exception = None
104
105        for attempt in range(self.settings.retries + 1):
106            try:
107                logger.debug(f"Fetching {url} (attempt {attempt + 1})")
108
109                response = self.session.get(url, timeout=self.settings.timeout)
110
111                # Handle rate limiting
112                if response.status_code == 429:
113                    retry_after = int(response.headers.get("Retry-After", 60))
114                    raise RateLimitError(
115                        f"Rate limited, retry after {retry_after}s", retry_after
116                    )
117
118                response.raise_for_status()
119
120                logger.debug(
121                    f"Successfully fetched {url} ({len(response.content)} bytes)"
122                )
123                return response.content
124
125            except requests.exceptions.RequestException as e:
126                last_exception = e
127                if attempt < self.settings.retries:
128                    sleep_time = self.settings.backoff * (2**attempt)
129                    logger.warning(
130                        f"Request failed (attempt {attempt + 1}): {e}. Retrying in {sleep_time}s"
131                    )
132                    time.sleep(sleep_time)
133                else:
134                    logger.error(f"All retry attempts failed for {url}")
135
136        raise FetchError(
137            f"Failed to fetch {url} after {self.settings.retries + 1} attempts: {last_exception}"
138        )
139
140    @staticmethod
141    def _parse_records(content: bytes, report_date: date) -> list[ShortRecord]:
142        """Apply the same parsing and validation to cached and downloaded data."""
143        records = parse_csv_content(content, report_date)
144        return [
145            ShortRecord(**record) for record in validate_records(records, report_date)
146        ]
147
148    def fetch_day(self, d: date, *, force: bool = False) -> FetchResult:
149        """Fetch short selling data for a single day.
150
151        Args:
152            d: Date to fetch data for
153            force: If True, bypass cache and fetch fresh data
154
155        Returns:
156            FetchResult containing the data and metadata
157
158        Raises:
159            NotFoundError: If data is not available for the date
160            FetchError: If there's an error fetching the data
161        """
162        logger.info(f"Fetching short selling data for {d}")
163        start_time = time.perf_counter()
164
165        # Check cache first (unless forced)
166        if not force:
167            cached_content = self.cache.read_cached(d)
168            if cached_content:
169                logger.debug(f"Using cached data for {d}")
170                record_models = self._parse_records(cached_content, d)
171                elapsed_ms = (time.perf_counter() - start_time) * 1000.0
172                return FetchResult(
173                    fetch_date=d,
174                    record_count=len(record_models),
175                    from_cache=True,
176                    fetch_time_ms=elapsed_ms,
177                    url=None,
178                    records=record_models,
179                )
180
181        # Resolve URL for the date
182        try:
183            url = self.resolver.url_for(d)
184        except NotFoundError:
185            logger.warning(f"No URL found for {d}")
186            raise
187
188        # Fetch content
189        content = self._fetch_with_retry(url)
190
191        record_models = self._parse_records(content, d)
192
193        # Cache the raw content
194        try:
195            self.cache.write_cached(d, content)
196        except Exception as e:
197            logger.warning(f"Failed to cache data for {d}: {e}")
198
199        logger.info(f"Successfully fetched {len(record_models)} records for {d}")
200        elapsed_ms = (time.perf_counter() - start_time) * 1000.0
201        return FetchResult(
202            fetch_date=d,
203            record_count=len(record_models),
204            from_cache=False,
205            fetch_time_ms=elapsed_ms,
206            url=url,
207            records=record_models,
208        )
209
210    def fetch_range(
211        self, start_date: date, end_date: date, *, force: bool = False
212    ) -> RangeResult:
213        """Fetch short selling data for a date range.
214
215        Args:
216            start_date: Start date (inclusive)
217            end_date: End date (inclusive)
218            force: If True, bypass cache and fetch fresh data
219
220        Returns:
221            RangeResult containing the data and metadata
222
223        Raises:
224            ValueError: If start_date > end_date
225            FetchError: If there's an error fetching data
226        """
227        if start_date > end_date:
228            raise ValueError("start_date must be <= end_date")
229
230        logger.info(f"Fetching short selling data from {start_date} to {end_date}")
231        overall_start = time.perf_counter()
232
233        results = {}
234        failed_dates = []
235        current_date = start_date
236
237        while current_date <= end_date:
238            try:
239                fetch_result = self.fetch_day(current_date, force=force)
240                results[current_date] = fetch_result
241            except NotFoundError:
242                logger.warning(f"No data available for {current_date}")
243                failed_dates.append(current_date)
244            except Exception as e:
245                logger.error(f"Failed to fetch data for {current_date}: {e}")
246                failed_dates.append(current_date)
247
248            current_date += timedelta(days=1)
249
250        total_records = sum(result.record_count for result in results.values())
251        logger.info(
252            f"Successfully fetched data for {len(results)} dates with {total_records} total records"
253        )
254
255        total_elapsed_ms = (time.perf_counter() - overall_start) * 1000.0
256        return RangeResult(
257            start_date=start_date,
258            end_date=end_date,
259            total_records=total_records,
260            successful_dates=list(results.keys()),
261            failed_dates=failed_dates,
262            total_fetch_time_ms=total_elapsed_ms,
263            results=results,
264        )
265
266    def clear_cache(self) -> None:
267        """Clear all cached files."""
268        logger.info("Clearing cache")
269        self.cache.clear_cache()
270
271    def cache_stats(self) -> CacheStats:
272        """Get cache statistics.
273
274        Returns:
275            CacheStats object with cache information
276        """
277        stats = self.cache.cache_stats()
278        return CacheStats(**stats)
279
280    def cleanup_cache(self, max_age_days: int = 30) -> None:
281        """Remove cached files older than max_age_days."""
282        logger.info(f"Cleaning up cache files older than {max_age_days} days")
283
284        cutoff_time = time.time() - (max_age_days * 24 * 3600)
285        cache_dir = self.cache.cache_dir
286
287        removed_count = 0
288        for cache_file in cache_dir.glob("*.csv"):
289            if cache_file.stat().st_mtime < cutoff_time:
290                try:
291                    cache_file.unlink()
292                    removed_count += 1
293                    logger.debug(f"Removed old cache file: {cache_file}")
294                except OSError as e:
295                    logger.warning(f"Failed to remove {cache_file}: {e}")
296
297        logger.info(f"Removed {removed_count} old cache files")
298
299        # Also cleanup stale locks
300        self.cache.cleanup_stale_locks()
301
302    def __enter__(self) -> "ShortsClient":
303        """Context manager entry."""
304        return self
305
306    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
307        """Context manager exit - cleanup resources."""
308        if hasattr(self.session, "close"):
309            self.session.close()

 7class FetchError(Exception):
 8    """Base exception for fetch operations."""
 9
10    def __init__(
11        self,
12        message: str,
13        date_requested: date | None = None,
14        url: str | None = None,
15    ):
16        """Initialize FetchError.
17
18        Args:
19            message: Error message.
20            date_requested: Date that was being fetched when error occurred.
21            url: URL that was being accessed when error occurred.
22        """
23        super().__init__(message)
24        self.date_requested = date_requested
25        self.url = url

28class NotFoundError(FetchError):
29    """Data not found for specified date."""
30
31    def __init__(self, date_requested: date, url: str | None = None):
32        """Initialize NotFoundError.
33
34        Args:
35            date_requested: Date for which data was not found.
36            url: URL that was checked for data.
37        """
38        message = f"No data found for date {date_requested.strftime('%Y-%m-%d')}"
39        if url:
40            message += f" at URL: {url}"
41        super().__init__(message, date_requested, url)

44class RateLimitError(FetchError):
45    """Rate limit exceeded."""
46
47    def __init__(
48        self, message: str = "Rate limit exceeded", retry_after: int | None = None
49    ):
50        """Initialize RateLimitError.
51
52        Args:
53            message: Error message.
54            retry_after: Number of seconds to wait before retrying.
55        """
56        super().__init__(message)
57        self.retry_after = retry_after

60class ParseError(FetchError):
61    """CSV parsing failed."""
62
63    def __init__(
64        self,
65        message: str,
66        date_requested: date | None = None,
67        details: str | None = None,
68    ):
69        """Initialize ParseError.
70
71        Args:
72            message: Error message.
73            date_requested: Date being parsed when error occurred.
74            details: Additional error details.
75        """
76        full_message = message
77        if details:
78            full_message += f" Details: {details}"
79        super().__init__(full_message, date_requested)
80        self.details = details

83class CacheError(FetchError):
84    """Cache operation failed."""
85
86    def __init__(self, message: str, cache_path: str | None = None):
87        """Initialize CacheError.
88
89        Args:
90            message: Error message.
91            cache_path: Path to cache file that caused the error.
92        """
93        super().__init__(message)
94        self.cache_path = cache_path

 74class PandasAdapter:
 75    """Adapter for pandas DataFrame integration."""
 76
 77    def __init__(self, client: ShortsClient):
 78        """Initialize with a ShortsClient instance."""
 79        self.client = client
 80        if pd is None:
 81            raise ImportError(
 82                "pandas is required for PandasAdapter. "
 83                "Install with: pip install 'asxshorts[pandas]'"
 84            )
 85        self.pd = pd  # type: ignore[assignment]
 86
 87    def fetch_day_df(self, d: date, *, force: bool = False) -> "pd.DataFrame":
 88        """Fetch data for a single date as pandas DataFrame.
 89
 90        Args:
 91            d: Target date
 92            force: Bypass cache if True
 93
 94        Returns:
 95            pandas DataFrame with short selling data
 96        """
 97        result = self.client.fetch_day(d, force=force)
 98        records_dict = [record.model_dump() for record in result.records]
 99        return self._records_to_dataframe(records_dict)
100
101    def fetch_range_df(
102        self, start: date, end: date, *, force: bool = False
103    ) -> "pd.DataFrame":
104        """Fetch data for a date range as pandas DataFrame.
105
106        Args:
107            start: Start date (inclusive)
108            end: End date (inclusive)
109            force: Bypass cache if True
110
111        Returns:
112            pandas DataFrame with short selling data
113        """
114        result = self.client.fetch_range(start, end, force=force)
115        all_records = []
116        for fetch_result in result.results.values():
117            all_records.extend([record.model_dump() for record in fetch_result.records])
118        return self._records_to_dataframe(all_records)
119
120    def _records_to_dataframe(self, records: list[dict[str, Any]]) -> "pd.DataFrame":
121        """Convert records using the shared conversion helper."""
122        return to_pandas(records)

258def create_pandas_adapter(client: ShortsClient | None = None) -> PandasAdapter:
259    """Create a PandasAdapter with optional client.
260
261    Args:
262        client: ShortsClient instance, creates default if None
263
264    Returns:
265        PandasAdapter instance
266    """
267    if client is None:
268        client = ShortsClient()
269    return PandasAdapter(client)

125class PolarsAdapter:
126    """Adapter for polars DataFrame integration."""
127
128    def __init__(self, client: ShortsClient):
129        """Initialize with a ShortsClient instance."""
130        self.client = client
131        if pl is None:
132            raise ImportError(
133                "polars is required for PolarsAdapter. "
134                "Install with: pip install 'asxshorts[polars]'"
135            )
136        self.pl = pl  # type: ignore[assignment]
137
138    def fetch_day_df(self, d: date, *, force: bool = False) -> "pl.DataFrame":
139        """Fetch data for a single date as polars DataFrame.
140
141        Args:
142            d: Target date
143            force: Bypass cache if True
144
145        Returns:
146            polars DataFrame with short selling data
147        """
148        result = self.client.fetch_day(d, force=force)
149        records_dict = [record.model_dump() for record in result.records]
150        return self._records_to_dataframe(records_dict)
151
152    def fetch_range_df(
153        self, start: date, end: date, *, force: bool = False
154    ) -> "pl.DataFrame":
155        """Fetch data for a date range as polars DataFrame.
156
157        Args:
158            start: Start date (inclusive)
159            end: End date (inclusive)
160            force: Bypass cache if True
161
162        Returns:
163            polars DataFrame with short selling data
164        """
165        result = self.client.fetch_range(start, end, force=force)
166        all_records = []
167        for fetch_result in result.results.values():
168            all_records.extend([record.model_dump() for record in fetch_result.records])
169        return self._records_to_dataframe(all_records)
170
171    def _records_to_dataframe(self, records: list[dict[str, Any]]) -> "pl.DataFrame":
172        """Convert records using the shared conversion helper."""
173        return to_polars(records)

272def create_polars_adapter(client: ShortsClient | None = None) -> PolarsAdapter:
273    """Create a PolarsAdapter with optional client.
274
275    Args:
276        client: ShortsClient instance, creates default if None
277
278    Returns:
279        PolarsAdapter instance
280    """
281    if client is None:
282        client = ShortsClient()
283    return PolarsAdapter(client)