mcmaster_scraper.async_api

View Source

 1from asyncio import create_task, gather
 2
 3from pandas import DataFrame
 4
 5from ._api.scraper import get_product_api_response
 6from ._api.table_parser import get_products_table
 7from ._utils.cache import get_cached, set_cached
 8from ._utils.event_loop_wrapper import run_in_loop_async
 9
10
11async def get_products_from_url(url: str, refresh: bool = False) -> DataFrame:
12    """Gets product tables from a McMaster-Carr URL.
13
14    See Also
15    --------
16    sync_api.get_products_from_url
17    """
18    cached = get_cached(url)
19    if cached and not refresh:
20        data = cached
21    else:
22        data = await run_in_loop_async(get_product_api_response(url))
23        set_cached(url, data)
24
25    return get_products_table(data)
26
27
28async def get_products_from_urls(
29    urls: list[str], refresh: bool = False
30) -> list[DataFrame]:
31    """Gets product tables from a list of McMaster-Carr URLs.
32
33    See Also
34    --------
35    sync_api.get_products_from_urls
36    """
37    tasks = [create_task(get_products_from_url(url, refresh)) for url in urls]
38    return await gather(*tasks)

async def get_products_from_url(url: str, refresh: bool = False) -> pandas.DataFrame: View Source

12async def get_products_from_url(url: str, refresh: bool = False) -> DataFrame:
13    """Gets product tables from a McMaster-Carr URL.
14
15    See Also
16    --------
17    sync_api.get_products_from_url
18    """
19    cached = get_cached(url)
20    if cached and not refresh:
21        data = cached
22    else:
23        data = await run_in_loop_async(get_product_api_response(url))
24        set_cached(url, data)
25
26    return get_products_table(data)

Gets product tables from a McMaster-Carr URL.

29async def get_products_from_urls(
30    urls: list[str], refresh: bool = False
31) -> list[DataFrame]:
32    """Gets product tables from a list of McMaster-Carr URLs.
33
34    See Also
35    --------
36    sync_api.get_products_from_urls
37    """
38    tasks = [create_task(get_products_from_url(url, refresh)) for url in urls]
39    return await gather(*tasks)

Gets product tables from a list of McMaster-Carr URLs.