You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
318 lines
13 KiB
Python
318 lines
13 KiB
Python
# backend/app/connectors/mail/real_gmail.py — Gmail API sync 커넥터 (phase-13, phase-16)
|
|
# OAuth2 액세스 토큰으로 Gmail REST(messages.list + messages.get) 증분 호출 + messages.send 발송.
|
|
import base64
|
|
from datetime import UTC, datetime
|
|
from email.message import EmailMessage
|
|
|
|
import httpx
|
|
from sqlmodel import Session, select
|
|
|
|
from ...config import get_settings
|
|
from ...models import Email, ExternalLink
|
|
from ..base import BaseConnector, NormalizedRecord, RawRecord
|
|
from ..oauth import valid_access_token
|
|
from ..ratelimit import RateLimiter
|
|
from .normalize import (
|
|
folder_dict,
|
|
mail_account_id_for,
|
|
normalize_email,
|
|
replace_account_folders,
|
|
upsert_email,
|
|
)
|
|
|
|
API = "https://gmail.googleapis.com/gmail/v1"
|
|
|
|
# Gmail 카테고리 라벨 → 사이드바 폴더(slug, 한글명, 아이콘, 정렬).
|
|
_GMAIL_CATEGORIES = [
|
|
("cat:promotions", "프로모션", "ticket", 10),
|
|
("cat:social", "소셜", "users", 11),
|
|
("cat:updates", "업데이트", "bell", 12),
|
|
]
|
|
|
|
|
|
def _epoch_to_dt(ms):
|
|
return datetime.fromtimestamp(int(ms) / 1000, tz=UTC) if ms else None
|
|
|
|
|
|
class GmailConnector(BaseConnector):
|
|
domain = "mail"
|
|
entity_type = "email"
|
|
_rl = RateLimiter(rate=4, per=1.0) # Gmail per-user QPS 보수적 제한
|
|
|
|
def _labels_map(self, c, headers) -> dict[str, str]:
|
|
"""Gmail 사용자 라벨 id → 이름(메시지의 사용자 라벨 해석용)."""
|
|
self._rl.acquire()
|
|
r = c.get(f"{API}/users/me/labels", headers=headers)
|
|
self._rl.handle_response(r)
|
|
r.raise_for_status()
|
|
return {
|
|
L["id"]: L["name"]
|
|
for L in r.json().get("labels", [])
|
|
if L.get("type") == "user"
|
|
}
|
|
|
|
def _sync_folders(self, session, macct: str, label_map: dict[str, str]) -> None:
|
|
"""Gmail 라벨 구조 → AccountFolder 미러(받은편지함/카테고리/별표/라벨/스팸/휴지통 등)."""
|
|
folders: list[dict] = [folder_dict("inbox", "받은편지함", "inbox", 0)]
|
|
for slug, name, icon, sort in _GMAIL_CATEGORIES:
|
|
folders.append(folder_dict(slug, name, icon, sort, kind="category"))
|
|
folders.append(folder_dict("starred", "별표", "star", 40))
|
|
# 사용자 라벨(빈 라벨은 폴더 엔드포인트에서 동기화분 0 이면 숨김).
|
|
for lid, name in sorted(label_map.items(), key=lambda kv: kv[1].lower()):
|
|
folders.append(
|
|
folder_dict(f"label:{name}", name, "hash", 20, kind="label", provider_id=lid)
|
|
)
|
|
folders.append(folder_dict("sent", "보낸편지함", "send", 50))
|
|
folders.append(folder_dict("drafts", "임시보관함", "pen", 55))
|
|
folders.append(folder_dict("spam", "스팸", "alert", 60))
|
|
folders.append(folder_dict("trash", "휴지통", "trash", 65))
|
|
replace_account_folders(session, macct, folders)
|
|
|
|
def fetch(self, session: Session, *, full: bool = False):
|
|
st = self._state_row(session)
|
|
token = valid_access_token(session, self.account) # token_expired 면 예외 → sync error
|
|
headers = {"Authorization": f"Bearer {token}"}
|
|
cfg = get_settings()
|
|
macct = mail_account_id_for(self.account.external_account_id)
|
|
with httpx.Client(timeout=cfg.connector_http_timeout) as c:
|
|
label_map = self._labels_map(c, headers)
|
|
self._sync_folders(session, macct, label_map)
|
|
# 증분: history API 로 '변경된' 메시지만 재조회 → 신규+읽음/라벨까지 반영.
|
|
if not full and st.page_token:
|
|
changed, deleted, new_hist = self._history_changes(c, headers, st.page_token)
|
|
if new_hist is not None:
|
|
self._purge(session, deleted)
|
|
for mid in changed:
|
|
rec = self._get_raw(c, headers, mid, label_map)
|
|
if rec:
|
|
yield rec
|
|
st.page_token = new_hist
|
|
st.cursor = str(int(datetime.now(UTC).timestamp()))
|
|
session.add(st)
|
|
session.commit()
|
|
return
|
|
# new_hist None = historyId 만료 → 아래 윈도우 폴백으로 재동기화.
|
|
# full / 최초 / 만료: 받은편지함(+휴지통) 윈도우 수집 후 현재 historyId 캡처.
|
|
# 스팸은 가져오지 않는다(유저 요청 — 이미 스팸 분류된 건 수집 제외).
|
|
scopes = [("in:inbox", cfg.sync_max_messages)]
|
|
if full:
|
|
scopes.append(("in:trash", cfg.sync_page_size))
|
|
for base_q, cap in scopes:
|
|
if full:
|
|
q = base_q
|
|
elif st.cursor:
|
|
q = f"{base_q} after:{st.cursor}"
|
|
else:
|
|
q = f"{base_q} newer_than:7d"
|
|
for mid in self._list_ids(c, headers, q, cap, cfg):
|
|
rec = self._get_raw(c, headers, mid, label_map)
|
|
if rec:
|
|
yield rec
|
|
st.page_token = self._current_history_id(c, headers) or st.page_token
|
|
st.cursor = str(int(datetime.now(UTC).timestamp()))
|
|
session.add(st)
|
|
session.commit()
|
|
|
|
def _list_ids(self, c, headers, q, cap, cfg) -> list[str]:
|
|
"""messages.list 를 nextPageToken 따라 cap 까지 페이지네이션해 메시지 id 수집."""
|
|
ids: list[str] = []
|
|
page_token = None
|
|
while len(ids) < cap:
|
|
params = {"maxResults": min(cfg.sync_page_size, cap - len(ids)), "q": q}
|
|
if page_token:
|
|
params["pageToken"] = page_token
|
|
self._rl.acquire()
|
|
r = c.get(f"{API}/users/me/messages", headers=headers, params=params)
|
|
self._rl.handle_response(r)
|
|
r.raise_for_status()
|
|
body = r.json()
|
|
ids.extend(m["id"] for m in body.get("messages", []))
|
|
page_token = body.get("nextPageToken")
|
|
if not page_token:
|
|
break
|
|
return ids[:cap]
|
|
|
|
def _get_raw(self, c, headers, mid, label_map) -> RawRecord | None:
|
|
"""messages.get(full) → RawRecord. 사용자 라벨 주입. 삭제(404)면 None."""
|
|
self._rl.acquire()
|
|
d = c.get(f"{API}/users/me/messages/{mid}", headers=headers, params={"format": "full"})
|
|
if d.status_code == 404:
|
|
return None
|
|
self._rl.handle_response(d)
|
|
d.raise_for_status()
|
|
msg = d.json()
|
|
msg["_userLabels"] = [
|
|
label_map[lid] for lid in msg.get("labelIds", []) if lid in label_map
|
|
]
|
|
return RawRecord(
|
|
external_id=msg["id"],
|
|
payload=msg,
|
|
etag=str(msg.get("historyId", "")),
|
|
external_updated_at=_epoch_to_dt(msg.get("internalDate")),
|
|
)
|
|
|
|
def _history_changes(self, c, headers, start_id: str):
|
|
"""users.history.list → (변경된 msg id 집합, 삭제된 id 집합, 새 historyId).
|
|
404(historyId 만료)면 (빈, 빈, None) 반환 → 호출부가 윈도우 폴백."""
|
|
changed: set[str] = set()
|
|
deleted: set[str] = set()
|
|
new_hist = start_id
|
|
page = None
|
|
while True:
|
|
params = {
|
|
"startHistoryId": start_id,
|
|
"historyTypes": ["messageAdded", "messageDeleted", "labelAdded", "labelRemoved"],
|
|
"maxResults": 500,
|
|
}
|
|
if page:
|
|
params["pageToken"] = page
|
|
self._rl.acquire()
|
|
r = c.get(f"{API}/users/me/history", headers=headers, params=params)
|
|
if r.status_code == 404:
|
|
return set(), set(), None
|
|
self._rl.handle_response(r)
|
|
r.raise_for_status()
|
|
body = r.json()
|
|
for h in body.get("history", []):
|
|
for item in h.get("messagesDeleted", []):
|
|
deleted.add(item["message"]["id"])
|
|
for key in ("messagesAdded", "labelsAdded", "labelsRemoved"):
|
|
for item in h.get(key, []):
|
|
changed.add(item["message"]["id"])
|
|
new_hist = body.get("historyId", new_hist)
|
|
page = body.get("nextPageToken")
|
|
if not page:
|
|
break
|
|
changed -= deleted
|
|
return changed, deleted, new_hist
|
|
|
|
def _current_history_id(self, c, headers) -> str:
|
|
"""users.getProfile → 현재 historyId(증분 sync 시작점)."""
|
|
self._rl.acquire()
|
|
r = c.get(f"{API}/users/me/profile", headers=headers)
|
|
if r.status_code != 200:
|
|
return ""
|
|
return str(r.json().get("historyId", ""))
|
|
|
|
def _purge(self, session: Session, deleted_ids: set[str]) -> None:
|
|
"""서버에서 영구삭제된 메시지를 로컬에서도 제거(Email + 링크)."""
|
|
for ext in deleted_ids:
|
|
link = session.exec(
|
|
select(ExternalLink).where(
|
|
ExternalLink.account_id == self.account.id,
|
|
ExternalLink.external_id == ext,
|
|
ExternalLink.entity_type == "email",
|
|
)
|
|
).first()
|
|
if not link:
|
|
continue
|
|
e = session.get(Email, link.entity_id)
|
|
if e:
|
|
session.delete(e)
|
|
session.delete(link)
|
|
if deleted_ids:
|
|
session.commit()
|
|
|
|
def normalize(self, raw: RawRecord) -> NormalizedRecord:
|
|
return normalize_email(self.account, raw, provider="gmail")
|
|
|
|
def write(self, session: Session, norm: NormalizedRecord):
|
|
return upsert_email(session, norm)
|
|
|
|
def event_for(self, norm, entity_id):
|
|
return "mail.received"
|
|
|
|
def modify_labels(
|
|
self,
|
|
session: Session,
|
|
external_id: str,
|
|
*,
|
|
add: list[str] | None = None,
|
|
remove: list[str] | None = None,
|
|
) -> None:
|
|
"""별표(STARRED)·읽음(UNREAD)·보관(INBOX) 라벨을 Gmail 에 실제 반영."""
|
|
token = valid_access_token(session, self.account)
|
|
with httpx.Client(timeout=get_settings().connector_http_timeout) as c:
|
|
self._rl.acquire()
|
|
r = c.post(
|
|
f"{API}/users/me/messages/{external_id}/modify",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
json={"addLabelIds": add or [], "removeLabelIds": remove or []},
|
|
)
|
|
self._rl.handle_response(r)
|
|
r.raise_for_status()
|
|
|
|
def trash(self, session: Session, external_id: str) -> None:
|
|
"""메일을 Gmail 휴지통으로 이동(삭제)."""
|
|
token = valid_access_token(session, self.account)
|
|
with httpx.Client(timeout=get_settings().connector_http_timeout) as c:
|
|
self._rl.acquire()
|
|
r = c.post(
|
|
f"{API}/users/me/messages/{external_id}/trash",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
)
|
|
self._rl.handle_response(r)
|
|
if r.status_code != 404: # 이미 없으면 성공으로 간주
|
|
r.raise_for_status()
|
|
|
|
def get_attachment(
|
|
self, session: Session, external_id: str, attachment_id: str
|
|
) -> bytes:
|
|
"""Gmail 첨부 본문(bytes) 다운로드."""
|
|
token = valid_access_token(session, self.account)
|
|
with httpx.Client(timeout=get_settings().connector_http_timeout) as c:
|
|
self._rl.acquire()
|
|
r = c.get(
|
|
f"{API}/users/me/messages/{external_id}/attachments/{attachment_id}",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
)
|
|
self._rl.handle_response(r)
|
|
r.raise_for_status()
|
|
data = r.json().get("data", "")
|
|
return base64.urlsafe_b64decode(data + "===")
|
|
|
|
def send_mail(
|
|
self,
|
|
session: Session,
|
|
*,
|
|
to: str,
|
|
subject: str,
|
|
body: str,
|
|
cc: str = "",
|
|
bcc: str = "",
|
|
attachments: list[dict] | None = None,
|
|
) -> str:
|
|
token = valid_access_token(session, self.account)
|
|
cfg = get_settings()
|
|
msg = EmailMessage()
|
|
msg["To"] = to
|
|
if cc:
|
|
msg["Cc"] = cc
|
|
if bcc:
|
|
msg["Bcc"] = bcc
|
|
msg["Subject"] = subject
|
|
if self.account.external_account_id:
|
|
msg["From"] = self.account.external_account_id
|
|
msg.set_content(body)
|
|
for att in attachments or []:
|
|
raw_bytes = base64.b64decode(att.get("data_b64", ""))
|
|
mime = att.get("mime") or "application/octet-stream"
|
|
maintype, _, subtype = mime.partition("/")
|
|
msg.add_attachment(
|
|
raw_bytes,
|
|
maintype=maintype or "application",
|
|
subtype=subtype or "octet-stream",
|
|
filename=att.get("name", "attachment"),
|
|
)
|
|
raw = base64.urlsafe_b64encode(msg.as_bytes()).decode()
|
|
with httpx.Client(timeout=cfg.connector_http_timeout) as c:
|
|
self._rl.acquire()
|
|
r = c.post(
|
|
f"{API}/users/me/messages/send",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
json={"raw": raw},
|
|
)
|
|
self._rl.handle_response(r)
|
|
r.raise_for_status()
|
|
return "sent"
|