You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

vacuumlo.c 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. /*-------------------------------------------------------------------------
  2. *
  3. * vacuumlo.c
  4. * This removes orphaned large objects from a database.
  5. *
  6. * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
  7. * Portions Copyright (c) 1994, Regents of the University of California
  8. *
  9. *
  10. * IDENTIFICATION
  11. * contrib/vacuumlo/vacuumlo.c
  12. *
  13. *-------------------------------------------------------------------------
  14. */
  15. #include "postgres_fe.h"
  16. #include <sys/stat.h>
  17. #include <fcntl.h>
  18. #include <unistd.h>
  19. #ifdef HAVE_TERMIOS_H
  20. #include <termios.h>
  21. #endif
  22. #include "catalog/pg_class_d.h"
  23. #include "fe_utils/connect.h"
  24. #include "libpq-fe.h"
  25. #include "pg_getopt.h"
  26. #include "getopt_long.h"
  27. #define BUFSIZE 1024
  28. enum trivalue
  29. {
  30. TRI_DEFAULT,
  31. TRI_NO,
  32. TRI_YES
  33. };
  34. struct _param
  35. {
  36. char *pg_user;
  37. enum trivalue pg_prompt;
  38. char *pg_port;
  39. char *pg_host;
  40. const char *progname;
  41. int verbose;
  42. int dry_run;
  43. long transaction_limit;
  44. };
  45. static int vacuumlo(const char *database, const struct _param *param);
  46. static void usage(const char *progname);
  47. /*
  48. * This vacuums LOs of one database. It returns 0 on success, -1 on failure.
  49. */
  50. static int
  51. vacuumlo(const char *database, const struct _param *param)
  52. {
  53. PGconn *conn;
  54. PGresult *res,
  55. *res2;
  56. char buf[BUFSIZE];
  57. long matched;
  58. long deleted;
  59. int i;
  60. bool new_pass;
  61. bool success = true;
  62. static bool have_password = false;
  63. static char password[100];
  64. /* Note: password can be carried over from a previous call */
  65. if (param->pg_prompt == TRI_YES && !have_password)
  66. {
  67. simple_prompt("Password: ", password, sizeof(password), false);
  68. have_password = true;
  69. }
  70. /*
  71. * Start the connection. Loop until we have a password if requested by
  72. * backend.
  73. */
  74. do
  75. {
  76. #define PARAMS_ARRAY_SIZE 7
  77. const char *keywords[PARAMS_ARRAY_SIZE];
  78. const char *values[PARAMS_ARRAY_SIZE];
  79. keywords[0] = "host";
  80. values[0] = param->pg_host;
  81. keywords[1] = "port";
  82. values[1] = param->pg_port;
  83. keywords[2] = "user";
  84. values[2] = param->pg_user;
  85. keywords[3] = "password";
  86. values[3] = have_password ? password : NULL;
  87. keywords[4] = "dbname";
  88. values[4] = database;
  89. keywords[5] = "fallback_application_name";
  90. values[5] = param->progname;
  91. keywords[6] = NULL;
  92. values[6] = NULL;
  93. new_pass = false;
  94. conn = PQconnectdbParams(keywords, values, true);
  95. if (!conn)
  96. {
  97. fprintf(stderr, "Connection to database \"%s\" failed\n",
  98. database);
  99. return -1;
  100. }
  101. if (PQstatus(conn) == CONNECTION_BAD &&
  102. PQconnectionNeedsPassword(conn) &&
  103. !have_password &&
  104. param->pg_prompt != TRI_NO)
  105. {
  106. PQfinish(conn);
  107. simple_prompt("Password: ", password, sizeof(password), false);
  108. have_password = true;
  109. new_pass = true;
  110. }
  111. } while (new_pass);
  112. /* check to see that the backend connection was successfully made */
  113. if (PQstatus(conn) == CONNECTION_BAD)
  114. {
  115. fprintf(stderr, "Connection to database \"%s\" failed:\n%s",
  116. database, PQerrorMessage(conn));
  117. PQfinish(conn);
  118. return -1;
  119. }
  120. if (param->verbose)
  121. {
  122. fprintf(stdout, "Connected to database \"%s\"\n", database);
  123. if (param->dry_run)
  124. fprintf(stdout, "Test run: no large objects will be removed!\n");
  125. }
  126. res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL);
  127. if (PQresultStatus(res) != PGRES_TUPLES_OK)
  128. {
  129. fprintf(stderr, "Failed to set search_path:\n");
  130. fprintf(stderr, "%s", PQerrorMessage(conn));
  131. PQclear(res);
  132. PQfinish(conn);
  133. return -1;
  134. }
  135. PQclear(res);
  136. /*
  137. * First we create and populate the LO temp table
  138. */
  139. buf[0] = '\0';
  140. strcat(buf, "CREATE TEMP TABLE vacuum_l AS ");
  141. if (PQserverVersion(conn) >= 90000)
  142. strcat(buf, "SELECT oid AS lo FROM pg_largeobject_metadata");
  143. else
  144. strcat(buf, "SELECT DISTINCT loid AS lo FROM pg_largeobject");
  145. res = PQexec(conn, buf);
  146. if (PQresultStatus(res) != PGRES_COMMAND_OK)
  147. {
  148. fprintf(stderr, "Failed to create temp table:\n");
  149. fprintf(stderr, "%s", PQerrorMessage(conn));
  150. PQclear(res);
  151. PQfinish(conn);
  152. return -1;
  153. }
  154. PQclear(res);
  155. /*
  156. * Analyze the temp table so that planner will generate decent plans for
  157. * the DELETEs below.
  158. */
  159. buf[0] = '\0';
  160. strcat(buf, "ANALYZE vacuum_l");
  161. res = PQexec(conn, buf);
  162. if (PQresultStatus(res) != PGRES_COMMAND_OK)
  163. {
  164. fprintf(stderr, "Failed to vacuum temp table:\n");
  165. fprintf(stderr, "%s", PQerrorMessage(conn));
  166. PQclear(res);
  167. PQfinish(conn);
  168. return -1;
  169. }
  170. PQclear(res);
  171. /*
  172. * Now find any candidate tables that have columns of type oid.
  173. *
  174. * NOTE: we ignore system tables and temp tables by the expedient of
  175. * rejecting tables in schemas named 'pg_*'. In particular, the temp
  176. * table formed above is ignored, and pg_largeobject will be too. If
  177. * either of these were scanned, obviously we'd end up with nothing to
  178. * delete...
  179. *
  180. * NOTE: the system oid column is ignored, as it has attnum < 1. This
  181. * shouldn't matter for correctness, but it saves time.
  182. */
  183. buf[0] = '\0';
  184. strcat(buf, "SELECT s.nspname, c.relname, a.attname ");
  185. strcat(buf, "FROM pg_class c, pg_attribute a, pg_namespace s, pg_type t ");
  186. strcat(buf, "WHERE a.attnum > 0 AND NOT a.attisdropped ");
  187. strcat(buf, " AND a.attrelid = c.oid ");
  188. strcat(buf, " AND a.atttypid = t.oid ");
  189. strcat(buf, " AND c.relnamespace = s.oid ");
  190. strcat(buf, " AND t.typname in ('oid', 'lo') ");
  191. strcat(buf, " AND c.relkind in (" CppAsString2(RELKIND_RELATION) ", " CppAsString2(RELKIND_MATVIEW) ")");
  192. strcat(buf, " AND s.nspname !~ '^pg_'");
  193. res = PQexec(conn, buf);
  194. if (PQresultStatus(res) != PGRES_TUPLES_OK)
  195. {
  196. fprintf(stderr, "Failed to find OID columns:\n");
  197. fprintf(stderr, "%s", PQerrorMessage(conn));
  198. PQclear(res);
  199. PQfinish(conn);
  200. return -1;
  201. }
  202. for (i = 0; i < PQntuples(res); i++)
  203. {
  204. char *schema,
  205. *table,
  206. *field;
  207. schema = PQgetvalue(res, i, 0);
  208. table = PQgetvalue(res, i, 1);
  209. field = PQgetvalue(res, i, 2);
  210. if (param->verbose)
  211. fprintf(stdout, "Checking %s in %s.%s\n", field, schema, table);
  212. schema = PQescapeIdentifier(conn, schema, strlen(schema));
  213. table = PQescapeIdentifier(conn, table, strlen(table));
  214. field = PQescapeIdentifier(conn, field, strlen(field));
  215. if (!schema || !table || !field)
  216. {
  217. fprintf(stderr, "%s", PQerrorMessage(conn));
  218. PQclear(res);
  219. PQfinish(conn);
  220. if (schema != NULL)
  221. PQfreemem(schema);
  222. if (schema != NULL)
  223. PQfreemem(table);
  224. if (schema != NULL)
  225. PQfreemem(field);
  226. return -1;
  227. }
  228. snprintf(buf, BUFSIZE,
  229. "DELETE FROM vacuum_l "
  230. "WHERE lo IN (SELECT %s FROM %s.%s)",
  231. field, schema, table);
  232. res2 = PQexec(conn, buf);
  233. if (PQresultStatus(res2) != PGRES_COMMAND_OK)
  234. {
  235. fprintf(stderr, "Failed to check %s in table %s.%s:\n",
  236. field, schema, table);
  237. fprintf(stderr, "%s", PQerrorMessage(conn));
  238. PQclear(res2);
  239. PQclear(res);
  240. PQfinish(conn);
  241. PQfreemem(schema);
  242. PQfreemem(table);
  243. PQfreemem(field);
  244. return -1;
  245. }
  246. PQclear(res2);
  247. PQfreemem(schema);
  248. PQfreemem(table);
  249. PQfreemem(field);
  250. }
  251. PQclear(res);
  252. /*
  253. * Now, those entries remaining in vacuum_l are orphans. Delete 'em.
  254. *
  255. * We don't want to run each delete as an individual transaction, because
  256. * the commit overhead would be high. However, since 9.0 the backend will
  257. * acquire a lock per deleted LO, so deleting too many LOs per transaction
  258. * risks running out of room in the shared-memory lock table. Accordingly,
  259. * we delete up to transaction_limit LOs per transaction.
  260. */
  261. res = PQexec(conn, "begin");
  262. if (PQresultStatus(res) != PGRES_COMMAND_OK)
  263. {
  264. fprintf(stderr, "Failed to start transaction:\n");
  265. fprintf(stderr, "%s", PQerrorMessage(conn));
  266. PQclear(res);
  267. PQfinish(conn);
  268. return -1;
  269. }
  270. PQclear(res);
  271. buf[0] = '\0';
  272. strcat(buf,
  273. "DECLARE myportal CURSOR WITH HOLD FOR SELECT lo FROM vacuum_l");
  274. res = PQexec(conn, buf);
  275. if (PQresultStatus(res) != PGRES_COMMAND_OK)
  276. {
  277. fprintf(stderr, "DECLARE CURSOR failed: %s", PQerrorMessage(conn));
  278. PQclear(res);
  279. PQfinish(conn);
  280. return -1;
  281. }
  282. PQclear(res);
  283. snprintf(buf, BUFSIZE, "FETCH FORWARD %ld IN myportal",
  284. param->transaction_limit > 0 ? param->transaction_limit : 1000L);
  285. deleted = 0;
  286. while (1)
  287. {
  288. res = PQexec(conn, buf);
  289. if (PQresultStatus(res) != PGRES_TUPLES_OK)
  290. {
  291. fprintf(stderr, "FETCH FORWARD failed: %s", PQerrorMessage(conn));
  292. PQclear(res);
  293. PQfinish(conn);
  294. return -1;
  295. }
  296. matched = PQntuples(res);
  297. if (matched <= 0)
  298. {
  299. /* at end of resultset */
  300. PQclear(res);
  301. break;
  302. }
  303. for (i = 0; i < matched; i++)
  304. {
  305. Oid lo = atooid(PQgetvalue(res, i, 0));
  306. if (param->verbose)
  307. {
  308. fprintf(stdout, "\rRemoving lo %6u ", lo);
  309. fflush(stdout);
  310. }
  311. if (param->dry_run == 0)
  312. {
  313. if (lo_unlink(conn, lo) < 0)
  314. {
  315. fprintf(stderr, "\nFailed to remove lo %u: ", lo);
  316. fprintf(stderr, "%s", PQerrorMessage(conn));
  317. if (PQtransactionStatus(conn) == PQTRANS_INERROR)
  318. {
  319. success = false;
  320. PQclear(res);
  321. break;
  322. }
  323. }
  324. else
  325. deleted++;
  326. }
  327. else
  328. deleted++;
  329. if (param->transaction_limit > 0 &&
  330. (deleted % param->transaction_limit) == 0)
  331. {
  332. res2 = PQexec(conn, "commit");
  333. if (PQresultStatus(res2) != PGRES_COMMAND_OK)
  334. {
  335. fprintf(stderr, "Failed to commit transaction:\n");
  336. fprintf(stderr, "%s", PQerrorMessage(conn));
  337. PQclear(res2);
  338. PQclear(res);
  339. PQfinish(conn);
  340. return -1;
  341. }
  342. PQclear(res2);
  343. res2 = PQexec(conn, "begin");
  344. if (PQresultStatus(res2) != PGRES_COMMAND_OK)
  345. {
  346. fprintf(stderr, "Failed to start transaction:\n");
  347. fprintf(stderr, "%s", PQerrorMessage(conn));
  348. PQclear(res2);
  349. PQclear(res);
  350. PQfinish(conn);
  351. return -1;
  352. }
  353. PQclear(res2);
  354. }
  355. }
  356. PQclear(res);
  357. }
  358. /*
  359. * That's all folks!
  360. */
  361. res = PQexec(conn, "commit");
  362. if (PQresultStatus(res) != PGRES_COMMAND_OK)
  363. {
  364. fprintf(stderr, "Failed to commit transaction:\n");
  365. fprintf(stderr, "%s", PQerrorMessage(conn));
  366. PQclear(res);
  367. PQfinish(conn);
  368. return -1;
  369. }
  370. PQclear(res);
  371. PQfinish(conn);
  372. if (param->verbose)
  373. {
  374. if (param->dry_run)
  375. fprintf(stdout, "\rWould remove %ld large objects from database \"%s\".\n",
  376. deleted, database);
  377. else if (success)
  378. fprintf(stdout,
  379. "\rSuccessfully removed %ld large objects from database \"%s\".\n",
  380. deleted, database);
  381. else
  382. fprintf(stdout, "\rRemoval from database \"%s\" failed at object %ld of %ld.\n",
  383. database, deleted, matched);
  384. }
  385. return ((param->dry_run || success) ? 0 : -1);
  386. }
  387. static void
  388. usage(const char *progname)
  389. {
  390. printf("%s removes unreferenced large objects from databases.\n\n", progname);
  391. printf("Usage:\n %s [OPTION]... DBNAME...\n\n", progname);
  392. printf("Options:\n");
  393. printf(" -l, --limit=LIMIT commit after removing each LIMIT large objects\n");
  394. printf(" -n, --dry-run don't remove large objects, just show what would be done\n");
  395. printf(" -v, --verbose write a lot of progress messages\n");
  396. printf(" -V, --version output version information, then exit\n");
  397. printf(" -?, --help show this help, then exit\n");
  398. printf("\nConnection options:\n");
  399. printf(" -h, --host=HOSTNAME database server host or socket directory\n");
  400. printf(" -p, --port=PORT database server port\n");
  401. printf(" -U, --username=USERNAME user name to connect as\n");
  402. printf(" -w, --no-password never prompt for password\n");
  403. printf(" -W, --password force password prompt\n");
  404. printf("\n");
  405. printf("Report bugs to <pgsql-bugs@postgresql.org>.\n");
  406. }
  407. int
  408. main(int argc, char **argv)
  409. {
  410. static struct option long_options[] = {
  411. {"host", required_argument, NULL, 'h'},
  412. {"limit", required_argument, NULL, 'l'},
  413. {"dry-run", no_argument, NULL, 'n'},
  414. {"port", required_argument, NULL, 'p'},
  415. {"username", required_argument, NULL, 'U'},
  416. {"verbose", no_argument, NULL, 'v'},
  417. {"version", no_argument, NULL, 'V'},
  418. {"no-password", no_argument, NULL, 'w'},
  419. {"password", no_argument, NULL, 'W'},
  420. {"help", no_argument, NULL, '?'},
  421. {NULL, 0, NULL, 0}
  422. };
  423. int rc = 0;
  424. struct _param param;
  425. int c;
  426. int port;
  427. const char *progname;
  428. int optindex;
  429. progname = get_progname(argv[0]);
  430. /* Set default parameter values */
  431. param.pg_user = NULL;
  432. param.pg_prompt = TRI_DEFAULT;
  433. param.pg_host = NULL;
  434. param.pg_port = NULL;
  435. param.progname = progname;
  436. param.verbose = 0;
  437. param.dry_run = 0;
  438. param.transaction_limit = 1000;
  439. /* Process command-line arguments */
  440. if (argc > 1)
  441. {
  442. if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
  443. {
  444. usage(progname);
  445. exit(0);
  446. }
  447. if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
  448. {
  449. puts("vacuumlo (PostgreSQL) " PG_VERSION);
  450. exit(0);
  451. }
  452. }
  453. while ((c = getopt_long(argc, argv, "h:l:np:U:vwW", long_options, &optindex)) != -1)
  454. {
  455. switch (c)
  456. {
  457. case '?':
  458. fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
  459. exit(1);
  460. case 'h':
  461. param.pg_host = pg_strdup(optarg);
  462. break;
  463. case 'l':
  464. param.transaction_limit = strtol(optarg, NULL, 10);
  465. if (param.transaction_limit < 0)
  466. {
  467. fprintf(stderr,
  468. "%s: transaction limit must not be negative (0 disables)\n",
  469. progname);
  470. exit(1);
  471. }
  472. break;
  473. case 'n':
  474. param.dry_run = 1;
  475. param.verbose = 1;
  476. break;
  477. case 'p':
  478. port = strtol(optarg, NULL, 10);
  479. if ((port < 1) || (port > 65535))
  480. {
  481. fprintf(stderr, "%s: invalid port number: %s\n", progname, optarg);
  482. exit(1);
  483. }
  484. param.pg_port = pg_strdup(optarg);
  485. break;
  486. case 'U':
  487. param.pg_user = pg_strdup(optarg);
  488. break;
  489. case 'v':
  490. param.verbose = 1;
  491. break;
  492. case 'w':
  493. param.pg_prompt = TRI_NO;
  494. break;
  495. case 'W':
  496. param.pg_prompt = TRI_YES;
  497. break;
  498. default:
  499. fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
  500. exit(1);
  501. }
  502. }
  503. /* No database given? Show usage */
  504. if (optind >= argc)
  505. {
  506. fprintf(stderr, "vacuumlo: missing required argument: database name\n");
  507. fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
  508. exit(1);
  509. }
  510. for (c = optind; c < argc; c++)
  511. {
  512. /* Work on selected database */
  513. rc += (vacuumlo(argv[c], &param) != 0);
  514. }
  515. return rc;
  516. }