curl_cffi Documentation

pip install curl_cffi --upgrade

brew install lexiforest/tap/curl-cffi

pip install curl_cffi --upgrade --pre

git clone https://github.com/lexiforest/curl_cffi/
cd curl_cffi
make preprocess
pip install .

curl-cffi get tls.browserleaks.com/json

# curl-cffi can be hard to type, use an alias if you want
alias imp=curl-cffi
imp get tls.browserleaks.com/json --impersonate chrome

import curl_cffi

# Notice the impersonate parameter
r = curl_cffi.get("https://tls.browserleaks.com/json", impersonate="chrome")

print(r.json())
# output: {..., "ja3n_hash": "aa56c057ad164ec4fdcb7a5a283be9fc", ...}
# the js3n fingerprint should be the same as target browser

# To keep using the latest browser version as `curl_cffi` updates,
# simply set impersonate="chrome" without specifying a version.
# Other similar values are: "safari" and "safari_ios"
r = curl_cffi.get("https://tls.browserleaks.com/json", impersonate="chrome")

# Use http/3 with impersonation
r = curl_cffi.get(
    "https://fp.impersonate.pro/api/http3",
    http_version="v3",
    impersonate="chrome"
)

# To pin a specific version, use version numbers together.
r = curl_cffi.get("https://tls.browserleaks.com/json", impersonate="chrome124")

# To impersonate other than browsers, bring your own ja3/akamai strings
# See examples directory for details.
r = curl_cffi.get("https://tls.browserleaks.com/json", ja3=..., akamai=...)

# http/socks proxies are supported
proxies = {"https": "http://localhost:3128"}
r = curl_cffi.get("https://tls.browserleaks.com/json", impersonate="chrome", proxies=proxies)

proxies = {"https": "socks://localhost:3128"}
r = curl_cffi.get("https://tls.browserleaks.com/json", impersonate="chrome", proxies=proxies)

s = curl_cffi.Session()

# httpbin is a http test website, this endpoint makes the server set cookies
s.get("https://httpbin.org/cookies/set/foo/bar")
print(s.cookies)
# <Cookies[<Cookie foo=bar for httpbin.org />]>

# retrieve cookies again to verify
r = s.get("https://httpbin.org/cookies")
print(r.json())
# {'cookies': {'foo': 'bar'}}

curl-cffi list

curl-cffi update

from curl_cffi import AsyncSession

async with AsyncSession() as s:
    r = await s.get("https://example.com")

import asyncio
from curl_cffi import AsyncSession

urls = [
    "https://google.com/",
    "https://facebook.com/",
    "https://twitter.com/",
]

async with AsyncSession() as s:
    tasks = []
    for url in urls:
        task = s.get(url)
        tasks.append(task)
    results = await asyncio.gather(*tasks)

from curl_cffi import WebSocket

def on_message(ws: WebSocket, message: str | bytes):
    print(message)

ws = WebSocket(on_message=on_message)
ws.run_forever("wss://api.gemini.com/v1/marketdata/BTCUSD")

import asyncio
from curl_cffi import AsyncSession

async with AsyncSession() as session:
    async with session.ws_connect("wss://echo.websocket.org") as ws:
        await asyncio.gather(*[ws.send_str("Hello, World!") for _ in range(10)])
        async for message in ws:
            print(message)

pip install aiohttp curl_cffi

import curl_cffi

curl_cffi.get(url, proxy="http://user:[email protected]:3128")

import curl_cffi

proxies = {
    "http": "http://localhost:3128",
    "https": "http://localhost:3128"
}
curl_cffi.get(url, proxies=proxies)

from datetime import timedelta
from curl_cffi import Session

with Session(cache=timedelta(minutes=10)) as s:
    first = s.get("https://example.com/api/users")
    second = s.get("https://example.com/api/users")  # served from cache

from datetime import timedelta
from curl_cffi import FileCacheBackend, Session

cache = FileCacheBackend(
    path="tests/.cache/http",
    expires=timedelta(hours=1),
)

with Session(cache=cache) as s:
    response = s.get("https://api.example.test/items")

from datetime import timedelta
from curl_cffi import FileCacheBackend, Session

cache = FileCacheBackend(
    expires=timedelta(minutes=10),
    ignored=["utm_source", "cache_buster"],
)

with Session(cache=cache) as s:
    s.get("https://example.com/items?item=1&utm_source=a")
    s.get("https://example.com/items?item=1&utm_source=b")  # same cache entry

cache.clear()

from datetime import timedelta
from curl_cffi import CacheBackend, Session

class DictCache(CacheBackend):
    def __init__(self):
        super().__init__(expires=timedelta(minutes=10))
        self.data = {}

    def _read_payload(self, key):
        return self.data.get(key)

    def _write_payload(self, key, payload):
        self.data[key] = payload

    def _delete_payload(self, key):
        self.data.pop(key, None)

    def clear(self):
        self.data.clear()

with Session(cache=DictCache()) as s:
    response = s.get("https://example.com/api")

from curl_cffi import Curl, CurlOpt
from io import BytesIO

buffer = BytesIO()
c = Curl()
c.setopt(CurlOpt.URL, b'https://tls.browserleaks.com/json')
c.setopt(CurlOpt.WRITEDATA, buffer)

c.impersonate("chrome124")

c.perform()
c.close()
body = buffer.getvalue()
print(body.decode())

from curl_cffi import CurlHttpVersion, CurlOpt, requests

response = requests.get(
    "https://example.com",
    curl_options={CurlOpt.HTTP_VERSION: CurlHttpVersion.V1_1},
)

>>> from curl_cffi import CurlHttpVersion
>>> r = curl_cffi.get("https://example.com", http_version="v2")
>>> r.http_version == CurlHttpVersion.V2_0
True

.. autoclass:: curl_cffi.Curl

   .. automethod:: __init__
   .. automethod:: debug
   .. automethod:: setopt
   .. automethod:: getinfo
   .. automethod:: version
   .. automethod:: impersonate
   .. automethod:: perform
   .. automethod:: duphandle
   .. automethod:: reset
   .. automethod:: parse_cookie_headers
   .. automethod:: get_reason_phrase
   .. automethod:: parse_status_line
   .. automethod:: close
   .. automethod:: ws_recv
   .. automethod:: ws_send
   .. automethod:: ws_close

AsyncCurl

.. autoclass:: curl_cffi.CurlMime

   .. automethod:: __init__
   .. automethod:: addpart
   .. automethod:: from_list
   .. automethod:: attach
   .. automethod:: close

Constants

``requests.get``, ``requests.post``, etc are just aliases of ``.request(METHOD, ...)``

.. autofunction:: curl_cffi.requests.request


Sessions
~~~~~~~

.. autoclass:: curl_cffi.requests.Session

   .. automethod:: __init__
   .. automethod:: request
   .. automethod:: stream
   .. automethod:: ws_connect


.. autoclass:: curl_cffi.requests.AsyncSession

   .. automethod:: __init__
   .. automethod:: request
   .. automethod:: stream
   .. automethod:: close
   .. automethod:: ws_connect

Cache
~~~~~

.. autoclass:: curl_cffi.requests.CacheBackend

   .. automethod:: __init__
   .. automethod:: should_cache_request
   .. automethod:: should_store_response
   .. automethod:: get
   .. automethod:: set
   .. automethod:: delete
   .. automethod:: _read_payload
   .. automethod:: _write_payload
   .. automethod:: _delete_payload
   .. automethod:: clear

.. autoclass:: curl_cffi.requests.FileCacheBackend

   .. automethod:: __init__
   .. automethod:: clear

Headers
~~~~~~~

.. autoclass:: curl_cffi.requests.Headers

   .. autoproperty:: encoding
   .. automethod:: raw
   .. automethod:: multi_items
   .. automethod:: get
   .. automethod:: get_list
   .. automethod:: update
   .. automethod:: __getitem__
   .. automethod:: __setitem__
   .. automethod:: __delitem__

Cookies
~~~~~~~

.. autoclass:: curl_cffi.requests.Cookies

   .. automethod:: set
   .. automethod:: get
   .. automethod:: delete
   .. automethod:: clear
   .. automethod:: update
   .. automethod:: __getitem__
   .. automethod:: __setitem__
   .. automethod:: __delitem__

Request, Response
~~~~~~

.. autoclass:: curl_cffi.requests.Request

.. autoclass:: curl_cffi.requests.Response

   .. automethod:: raise_for_status
   .. automethod:: iter_lines
   .. automethod:: iter_content
   .. automethod:: json
   .. automethod:: close
   .. automethod:: aiter_lines
   .. automethod:: aiter_content
   .. automethod:: atext
   .. automethod:: acontent
   .. automethod:: aclose

Asyncio
-------

WebSocket
---------

.. autoclass:: curl_cffi.requests.WebSocket

   .. automethod:: __init__
   .. automethod:: connect
   .. automethod:: recv
   .. automethod:: recv_str
   .. automethod:: recv_json
   .. automethod:: send
   .. automethod:: send_binary
   .. automethod:: send_bytes
   .. automethod:: send_str
   .. automethod:: send_json
   .. automethod:: ping
   .. automethod:: run_forever
   .. automethod:: close

.. autoclass:: curl_cffi.requests.AsyncWebSocket

   .. automethod:: __init__
   .. automethod:: recv
   .. automethod:: recv_str
   .. automethod:: recv_json
   .. automethod:: send
   .. automethod:: send_binary
   .. automethod:: send_bytes
   .. automethod:: send_str
   .. automethod:: send_json
   .. automethod:: ping
   .. automethod:: close

Exceptions and Warnings
-----------------------

Exceptions

.. autoclass:: curl_cffi.requests.exceptions.RequestsWarning
.. autoclass:: curl_cffi.requests.exceptions.FileModeWarning
.. autoclass:: curl_cffi.requests.exceptions.RequestsDependencyWarning

---

## File: docs/cookies.rst

Cookies
=======

Save and load cookies
------

Do not use ``get_dict`` to dump and load cookies. Cookies are more than just plain
key-value pairs.

Using pickle:

.. code-block:: python

    # example from: https://github.com/encode/httpx/issues/895
    import pickle
    # import httpx
    import curl_cffi

    def save_cookies(client):
        with open("cookies.pk", "wb") as f:
            pickle.dump(client.cookies.jar._cookies, f)

    def load_cookies():
        if not os.path.isfile("cookies.pk"):
            return None
        with open("cookies.pk", "rb") as f:
            return pickle.load(f)

    # client = httpx.Client(cookies=load_cookies())
    client = curl_cffi.Session()
    client.get("https://httpbin.org/cookies/set/foo/bar")
    save_cookies(client)

    client = curl_cffi.Session()
    client.cookies.jar._cookies.update(load_cookies())
    print(client.cookies.get("foo"))


Using mozilla cookie jar:

See: https://github.com/lexiforest/curl_cffi/issues/381

TODO: expose libcurl's native cookies.txt support.


Discard cookies when using Session
----------------------------------

You may need to discard cookies when using sessions, especially when using ``AsyncSession``.

Use the ``discard_cookies`` option.


.. code-block:: python

    s = requests.Session()

    set_url = "https://httpbin.org/cookies"

    r = s.get(set_url)
    assert r.cookies["foo"] == s.cookies["foo"]
    old_cookie = r.cookies["foo"]

    # Let's start discarding cookies
    s.discard_cookies = True
    r = s.get(set_url)
    assert r.cookies["foo"] != s.cookies["foo"]
    assert old_cookie == s.cookies["foo"]

    # The behavior can be reverted
    s.discard_cookies = False
    r = s.get(set_url)
    assert r.cookies["foo"] == s.cookies["foo"]
    old_cookie = r.cookies["foo"]

    # Also works as request parameter
    r = s.get(set_url, discard_cookies=True)
    assert r.cookies["foo"] != s.cookies["foo"]
    assert old_cookie == s.cookies["foo"]

---

## File: docs/dev.rst

Development and contributing
==============

This page documents how to compile curl-impersonate and curl-cffi from source. If binary
package is not available on your platform, you may refer to this page for some inspirations.

First, you need to check if there are libcurl-impersonate binaries for you platform. If
so, you can simply download and install them.

For now, a pre-compiled `libcurl-impersonate` is downloaded from github and built
into a bdist wheel, which is a binary package format used by PyPI. However, the
right way is to download curl and curl-impersonate sources on our side and compile
them all together.

By default, source builds download the platform-specific library into a temporary
directory. Set ``IMPERSONATE_BUILD_DIR`` to reuse an existing download or choose
where it is downloaded and unpacked. The directory must contain the library filename
configured for the current platform, such as ``libcurl-impersonate.so`` or
``libcurl-impersonate.a``::

    IMPERSONATE_BUILD_DIR=/path/to/lib pip install .

The default link type is configured per platform. Set ``IMPERSONATE_LINK_TYPE``
to ``dynamic`` or ``static`` to override it. A dynamic local build can be used
without merging its static dependency archives::

    IMPERSONATE_BUILD_DIR=/path/to/lib \
        IMPERSONATE_LINK_TYPE=dynamic pip install -e .

On macOS, the directory must contain ``libcurl-impersonate.dylib`` and any
versioned symlinks required by its install name. The directory is recorded as a
runtime search path in the extension module.

macOS

To install the local editable build:

.. code-block:: shell

    # This is for using the libcurl-impersonate built by GitHub actions

    sudo mkdir /Users/runner
    sudo chmod 777 /Users/runner

    # Dependencies

    brew install libidn2 zstd

    # Then install

    pip install -e .[test]
    pip install -e .[dev]

Contributing
------------

When opening PR, please do not use the ``main`` branch in your fork, otherwise I cannot
add my modification, such as unittests.

---

## File: docs/faq.rst

FAQ
==========================

What does the pro version offer? Is the open source project still maintained?
----------------------------------------------------------------------------

Yes, the open source project is maintained as before.

In the `pro version <https://impersonate.pro>`_, we provide:

- weekly update of targets
- profiles for mobile browsers and apps
- some private detection fields
- http/3 fingerprints and proxy support

And, a better financial situation will help the open source version better maintained.

Why does the JA3 fingerprints change for Chrome 110+ impersonation?
------

This is intended.

Chrome introduces ``ClientHello`` permutation in version 110, which means the order of
extensions will be random, thus JA3 fingerprints will be random. So, when comparing
JA3 fingerprints of ``curl_cffi`` and a browser, they may differ. However, this does not
mean that TLS fingerprints will not be a problem, ``ClientHello`` extension order is just
one factor of how servers can tell automated requests from browsers.

Roughly, this can be mitigated like:

.. code-block::

    ja3 = md5(list(extensions), ...other arguments)
    ja3n = md5(set(extensions), ...other arguments)

See more from `this article <https://www.fastly.com/blog/a-first-look-at-chromes-tls-clienthello-permutation-in-the-wild>`_
and `curl-impersonate notes <https://github.com/lwthiker/curl-impersonate/pull/148>`_.

Can I bypass Cloudflare with this project? or any other specific site.
------

Short answer is: it depends.

TLS and http2 fingerprints are just one of the many factors Cloudflare considers. Other
factors include but are not limited to: IP quality, request rate, JS fingerprints, etc.

There are different protection levels for website owners to choose. For the most basic
ones, TLS fingerprints alone maybe enough, but for higher levels, you may need to find
a better proxy IP provider and use browser automation tools like playwright.

If you are in a hurry or just want the professionals to take care of the hard parts,
you can consider the commercial solutions from our sponsors:

- `Yescaptcha <https://yescaptcha.com/i/stfnIO>`_, captcha resolver and proxy service for bypassing Cloudflare.
- `ScrapeNinja <https://scrapeninja.net/?utm_source=github&utm_medium=banner&utm_campaign=cffi>`_, Managed web scraping API.

For details, see the `Sponsor` section on front page.


I'm getting certs errors
------

The simplest way is to turn off cert verification by ``verify=False``:

.. code-block:: python

    r = curl_cffi.get("https://example.com", verify=False)


ErrCode: 77, Reason: error setting certificate verify locations
------

On Windows, if your Python environment or CA bundle path contains non-ASCII characters
(e.g. accents), libcurl may fail to open the CA file when passed as a narrow ``char*``.
``curl_cffi`` now encodes file-path options (e.g. ``CAINFO``, ``PROXY_CAINFO``,
``SSLCERT``) using the system's preferred ANSI code page on Windows to ensure correct
file access. This fixes most occurrences of error 77.

How to use with fiddler/charles to intercept content
------

Fiddler and Charles uses man-in-the-middle self-signed certs to intercept TLS traffic,
to use with them, simply set ``verify=False``.


ErrCode: 92, Reason: 'HTTP/2 stream 0 was not closed cleanly: PROTOCOL_ERROR (err 1)'
------

This error(http/2 stream 0) has been reported many times ever since `curl_cffi` was
published, but I still can not find a reproducible way to trigger it. Given that the
majority users are behind proxies, the situation is even more difficult to deal with.

I'm even not sure it's a bug introduced in libcurl, curl-impersonate or curl_cffi, or
it's just a server error. Depending on your context, here are some general suggestions
for you:

- First, try removing the ``Content-Length`` header from you request.
- Try to see if this error was caused by proxies, if so, use better proxies.
- If it stops working after a while, maybe you're just being blocked by, such as, Akamai.
- Force http/1.1 mode. Some websites' h2 implementation is simply broken.
- See if the url works in your real browser.
- Find a stable way to reproduce it, so we can finally fix, or at least bypass it.

To force curl to use http 1.1 only.

.. code-block:: python

    import curl_cffi

    r = curl_cffi.get("https://postman-echo.com", http_version=curl_cffi.CurlHttpVersion.V1_1)

Related issues:

- `#19 <https://github.com/lexiforest/curl_cffi/issues/19>`_, 
- `#42 <https://github.com/lexiforest/curl_cffi/issues/42>`_, 
- `#79 <https://github.com/lexiforest/curl_cffi/issues/79>`_, 
- `#165 <https://github.com/lexiforest/curl_cffi/issues/165>`_, 


Packaging with PyInstaller
------

If you encountered any issue with PyInstaller, here are a list of options provided by the
community:

Add the ``--hidden-import`` option.

.. code-block::

   pyinstaller -F .\example.py --hidden-import=_cffi_backend --collect-all curl_cffi

Add other paths:

.. code-block::

   pyinstaller --noconfirm --onefile --console \
       --paths "C:/Users/Administrator/AppData/Local/Programs/Python/Python39" \
       --add-data "C:/Users/Administrator/AppData/Local/Programs/Python/Python39/Lib/site-packages/curl_cffi.libs/libcurl-cbb416caa1dd01638554eab3f38d682d.dll;." \
       --collect-data "curl_cffi" \
       "C:/Users/Administrator/Desktop/test_script.py"


See also: 

- `#5 <https://github.com/lexiforest/curl_cffi/issues/5>`_
- `#48 <https://github.com/lexiforest/curl_cffi/issues/48>`_


How to change the order of headers?
------

By default, setting ``impersonate`` parameter will bring the corresponding headers. If
you want to change the order or use your own headers, you need to turn off that and bring
your own headers. For stored fingerprint targets, you can also load and edit a
``Fingerprint`` object directly.

.. code-block::

   fingerprint = curl_cffi.get_fingerprint("edge_146_macos_26")
   fingerprint.headers["User-Agent"] = "..."
   requests.get(url, impersonate=fingerprint)


How to deal with encoding/decoding errors?
------------------------------------------

Use ``chardet`` or ``cchardet``

.. code-block::

    >>> import curl_cffi
    >>> r = curl_cffi.get("https://example.com/messy_codec.html")
    >>> import chardet
    >>> chardet.detect(r.content)
    {'encoding': 'GB2312', 'confidence': 0.99, 'language': 'Chinese'}

Or use regex or lxml to parse the meta header:

.. code-block::

    <meta http-equiv="Content-Type" content="text/html; charset=gbk" />