00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025 #ifndef _PASSENGER_SPAWN_MANAGER_H_
00026 #define _PASSENGER_SPAWN_MANAGER_H_
00027
00028 #include <string>
00029 #include <vector>
00030 #include <boost/shared_ptr.hpp>
00031 #include <boost/thread.hpp>
00032 #include <boost/function.hpp>
00033 #include <oxt/system_calls.hpp>
00034 #include <oxt/backtrace.hpp>
00035
00036 #include <sys/types.h>
00037 #include <sys/wait.h>
00038 #include <sys/stat.h>
00039 #include <arpa/inet.h>
00040 #include <cstdio>
00041 #include <cstdarg>
00042 #include <unistd.h>
00043 #include <errno.h>
00044 #include <grp.h>
00045 #include <pwd.h>
00046 #include <signal.h>
00047
00048 #include "AbstractSpawnManager.h"
00049 #include "ServerInstanceDir.h"
00050 #include "FileDescriptor.h"
00051 #include "Constants.h"
00052 #include "MessageChannel.h"
00053 #include "AccountsDatabase.h"
00054 #include "RandomGenerator.h"
00055 #include "Exceptions.h"
00056 #include "Logging.h"
00057 #include "Utils/Base64.h"
00058 #include "Utils/SystemTime.h"
00059 #include "Utils/IOUtils.h"
00060
00061 namespace Passenger {
00062
00063 using namespace std;
00064 using namespace boost;
00065 using namespace oxt;
00066
00067
00068
00069
00070
00071
00072
00073
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094 class SpawnManager: public AbstractSpawnManager {
00095 private:
00096 static const int SERVER_SOCKET_FD = 3;
00097 static const int OWNER_SOCKET_FD = 4;
00098 static const int HIGHEST_FD = OWNER_SOCKET_FD;
00099
00100 string spawnServerCommand;
00101 ServerInstanceDir::GenerationPtr generation;
00102 AccountsDatabasePtr accountsDatabase;
00103 string rubyCommand;
00104 AnalyticsLoggerPtr analyticsLogger;
00105 int logLevel;
00106 string debugLogFile;
00107
00108 boost::mutex lock;
00109 RandomGenerator random;
00110
00111 pid_t pid;
00112 FileDescriptor ownerSocket;
00113 string socketFilename;
00114 string socketPassword;
00115 bool serverNeedsRestart;
00116
00117 static void deleteAccount(AccountsDatabasePtr accountsDatabase, const string &username) {
00118 accountsDatabase->remove(username);
00119 }
00120
00121
00122
00123
00124
00125
00126
00127
00128
00129 void restartServer() {
00130 TRACE_POINT();
00131 if (pid != 0) {
00132 UPDATE_TRACE_POINT();
00133 ownerSocket.close();
00134
00135
00136
00137
00138
00139 time_t begin = syscalls::time(NULL);
00140 bool done = false;
00141 while (!done && syscalls::time(NULL) - begin < 5) {
00142 if (syscalls::waitpid(pid, NULL, WNOHANG) > 0) {
00143 done = true;
00144 } else {
00145 syscalls::usleep(100000);
00146 }
00147 }
00148 UPDATE_TRACE_POINT();
00149 if (!done) {
00150 UPDATE_TRACE_POINT();
00151 P_TRACE(2, "Spawn server did not exit in time, killing it...");
00152 syscalls::kill(pid, SIGTERM);
00153 begin = syscalls::time(NULL);
00154 while (syscalls::time(NULL) - begin < 5) {
00155 if (syscalls::waitpid(pid, NULL, WNOHANG) > 0) {
00156 break;
00157 } else {
00158 syscalls::usleep(100000);
00159 }
00160 }
00161 }
00162 pid = 0;
00163 }
00164
00165 FileDescriptor serverSocket;
00166 string socketFilename;
00167 string socketPassword;
00168 int ret, fds[2];
00169
00170 UPDATE_TRACE_POINT();
00171 socketFilename = generation->getPath() + "/spawn-server/socket." +
00172 toString(getpid()) + "." +
00173 pointerToIntString(this);
00174 socketPassword = Base64::encode(random.generateByteString(32));
00175 serverSocket = createUnixServer(socketFilename.c_str());
00176 do {
00177 ret = chmod(socketFilename.c_str(), S_IRUSR | S_IWUSR | S_IXUSR);
00178 } while (ret == -1 && errno == EINTR);
00179 if (ret == -1) {
00180 int e = errno;
00181 syscalls::unlink(socketFilename.c_str());
00182 throw FileSystemException("Cannot set permissions on '" + socketFilename + "'",
00183 e, socketFilename);
00184 }
00185
00186 if (syscalls::socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == -1) {
00187 int e = errno;
00188 syscalls::unlink(socketFilename.c_str());
00189 throw SystemException("Cannot create a Unix socket", e);
00190 }
00191
00192 UPDATE_TRACE_POINT();
00193 pid = syscalls::fork();
00194 if (pid == 0) {
00195 dup2(serverSocket, HIGHEST_FD + 1);
00196 dup2(fds[1], HIGHEST_FD + 2);
00197 dup2(HIGHEST_FD + 1, SERVER_SOCKET_FD);
00198 dup2(HIGHEST_FD + 2, OWNER_SOCKET_FD);
00199
00200
00201 for (long i = sysconf(_SC_OPEN_MAX) - 1; i > HIGHEST_FD; i--) {
00202 close(i);
00203 }
00204
00205 execlp(rubyCommand.c_str(),
00206 rubyCommand.c_str(),
00207 spawnServerCommand.c_str(),
00208
00209
00210
00211
00212
00213
00214
00215 " ",
00216 (char *) NULL);
00217 int e = errno;
00218 fprintf(stderr, "*** Passenger ERROR (%s:%d):\n"
00219 "Could not start the spawn server: %s: %s (%d)\n",
00220 __FILE__, __LINE__,
00221 rubyCommand.c_str(), strerror(e), e);
00222 fflush(stderr);
00223 _exit(1);
00224 } else if (pid == -1) {
00225 int e = errno;
00226 syscalls::unlink(socketFilename.c_str());
00227 syscalls::close(fds[0]);
00228 syscalls::close(fds[1]);
00229 pid = 0;
00230 throw SystemException("Unable to fork a process", e);
00231 } else {
00232 FileDescriptor ownerSocket = fds[0];
00233 syscalls::close(fds[1]);
00234 serverSocket.close();
00235
00236
00237 writeExact(ownerSocket, socketFilename + "\n");
00238 writeExact(ownerSocket, socketPassword + "\n");
00239 writeExact(ownerSocket, generation->getPath() + "\n");
00240 if (analyticsLogger != NULL) {
00241 writeExact(ownerSocket, analyticsLogger->getAddress() + "\n");
00242 writeExact(ownerSocket, analyticsLogger->getUsername() + "\n");
00243 writeExact(ownerSocket, Base64::encode(analyticsLogger->getPassword()) + "\n");
00244 writeExact(ownerSocket, analyticsLogger->getNodeName() + "\n");
00245 } else {
00246 writeExact(ownerSocket, "\n");
00247 writeExact(ownerSocket, "\n");
00248 writeExact(ownerSocket, "\n");
00249 writeExact(ownerSocket, "\n");
00250 }
00251 writeExact(ownerSocket, toString(logLevel) + "\n");
00252 writeExact(ownerSocket, debugLogFile + "\n");
00253
00254 this->ownerSocket = ownerSocket;
00255 this->socketFilename = socketFilename;
00256 this->socketPassword = socketPassword;
00257 spawnServerStarted();
00258 }
00259 }
00260
00261
00262
00263
00264
00265
00266
00267
00268 FileDescriptor connect() const {
00269 TRACE_POINT();
00270 FileDescriptor fd = connectToUnixServer(socketFilename.c_str());
00271 MessageChannel channel(fd);
00272 UPDATE_TRACE_POINT();
00273 channel.writeScalar(socketPassword);
00274 return fd;
00275 }
00276
00277
00278
00279
00280
00281
00282
00283
00284
00285
00286 ProcessPtr sendSpawnCommand(const PoolOptions &options) {
00287 TRACE_POINT();
00288 FileDescriptor connection;
00289 MessageChannel channel;
00290
00291 P_DEBUG("Spawning a new application process for " << options.appRoot << "...");
00292
00293 try {
00294 connection = connect();
00295 channel = MessageChannel(connection);
00296 } catch (const SystemException &e) {
00297 throw SpawnException(string("Could not connect to the spawn server: ") +
00298 e.sys());
00299 } catch (const std::exception &e) {
00300 throw SpawnException(string("Could not connect to the spawn server: ") +
00301 e.what());
00302 }
00303
00304 UPDATE_TRACE_POINT();
00305 vector<string> args;
00306 string appRoot;
00307 pid_t appPid;
00308 int i, nServerSockets, ownerPipe;
00309 Process::SocketInfoMap serverSockets;
00310 string detachKey = random.generateAsciiString(43);
00311
00312
00313 string connectPassword = random.generateAsciiString(43);
00314 string gupid = integerToHex(SystemTime::get() / 60) + "-" +
00315 random.generateAsciiString(11);
00316 AccountPtr account;
00317 function<void ()> destructionCallback;
00318
00319 try {
00320 args.push_back("spawn_application");
00321 options.toVector(args);
00322
00323 args.push_back("detach_key");
00324 args.push_back(detachKey);
00325 args.push_back("connect_password");
00326 args.push_back(connectPassword);
00327 if (accountsDatabase != NULL) {
00328 string username = "_backend-" + toString(accountsDatabase->getUniqueNumber());
00329 string password = random.generateByteString(MESSAGE_SERVER_MAX_PASSWORD_SIZE);
00330 account = accountsDatabase->add(username, password, false, options.rights);
00331 destructionCallback = boost::bind(&SpawnManager::deleteAccount,
00332 accountsDatabase, username);
00333
00334 args.push_back("pool_account_username");
00335 args.push_back(username);
00336 args.push_back("pool_account_password_base64");
00337 args.push_back(Base64::encode(password));
00338 }
00339
00340 channel.write(args);
00341 } catch (const SystemException &e) {
00342 throw SpawnException(string("Could not write 'spawn_application' "
00343 "command to the spawn server: ") + e.sys());
00344 }
00345
00346 try {
00347 UPDATE_TRACE_POINT();
00348
00349 if (!channel.read(args)) {
00350 throw SpawnException("The spawn server has exited unexpectedly.");
00351 }
00352 if (args.size() != 1) {
00353 throw SpawnException("The spawn server sent an invalid message.");
00354 }
00355 if (args[0] == "error_page") {
00356 UPDATE_TRACE_POINT();
00357 string errorPage;
00358
00359 if (!channel.readScalar(errorPage)) {
00360 throw SpawnException("The spawn server has exited unexpectedly.");
00361 }
00362 throw SpawnException("An error occured while spawning the application.",
00363 errorPage);
00364 } else if (args[0] != "ok") {
00365 throw SpawnException("The spawn server sent an invalid message.");
00366 }
00367
00368
00369 UPDATE_TRACE_POINT();
00370 if (!channel.read(args)) {
00371 throw SpawnException("The spawn server has exited unexpectedly.");
00372 }
00373 if (args.size() != 3) {
00374 throw SpawnException("The spawn server sent an invalid message.");
00375 }
00376
00377 appRoot = args[0];
00378 appPid = (pid_t) stringToULL(args[1]);
00379 nServerSockets = atoi(args[2]);
00380
00381 UPDATE_TRACE_POINT();
00382 for (i = 0; i < nServerSockets; i++) {
00383 if (!channel.read(args)) {
00384 throw SpawnException("The spawn server has exited unexpectedly.");
00385 }
00386 if (args.size() != 3) {
00387 throw SpawnException("The spawn server sent an invalid message.");
00388 }
00389 serverSockets[args[0]] = Process::SocketInfo(args[1], args[2]);
00390 }
00391 if (serverSockets.find("main") == serverSockets.end()) {
00392 UPDATE_TRACE_POINT();
00393 throw SpawnException("The spawn server sent an invalid message.");
00394 }
00395 } catch (const SystemException &e) {
00396 throw SpawnException(string("Could not read from the spawn server: ") + e.sys());
00397 }
00398
00399 UPDATE_TRACE_POINT();
00400 try {
00401 ownerPipe = channel.readFileDescriptor();
00402 } catch (const SystemException &e) {
00403 throw SpawnException(string("Could not receive the spawned "
00404 "application's owner pipe from the spawn server: ") +
00405 e.sys());
00406 } catch (const IOException &e) {
00407 throw SpawnException(string("Could not receive the spawned "
00408 "application's owner pipe from the spawn server: ") +
00409 e.what());
00410 }
00411
00412 UPDATE_TRACE_POINT();
00413 P_DEBUG("Application process " << appPid << " spawned");
00414 return ProcessPtr(new Process(appRoot, appPid, ownerPipe, serverSockets,
00415 detachKey, connectPassword, gupid, destructionCallback));
00416 }
00417
00418
00419
00420
00421
00422 ProcessPtr
00423 handleSpawnException(const SpawnException &e, const PoolOptions &options) {
00424 TRACE_POINT();
00425 bool restarted;
00426 try {
00427 P_DEBUG("Spawn server died. Attempting to restart it...");
00428 this_thread::disable_syscall_interruption dsi;
00429 restartServer();
00430 P_DEBUG("Restart seems to be successful.");
00431 restarted = true;
00432 } catch (const IOException &e) {
00433 P_DEBUG("Restart failed: " << e.what());
00434 restarted = false;
00435 } catch (const SystemException &e) {
00436 P_DEBUG("Restart failed: " << e.what());
00437 restarted = false;
00438 }
00439 if (restarted) {
00440 return sendSpawnCommand(options);
00441 } else {
00442 throw SpawnException("The spawn server died unexpectedly, and restarting it failed.");
00443 }
00444 }
00445
00446
00447
00448
00449
00450
00451
00452
00453
00454 void sendReloadCommand(const string &appRoot) {
00455 TRACE_POINT();
00456 FileDescriptor connection;
00457 MessageChannel channel;
00458
00459 try {
00460 connection = connect();
00461 channel = MessageChannel(connection);
00462 } catch (SystemException &e) {
00463 e.setBriefMessage("Could not connect to the spawn server");
00464 throw;
00465 } catch (const RuntimeException &e) {
00466 throw RuntimeException(string("Could not connect to the spawn server: ") +
00467 e.what());
00468 }
00469
00470 try {
00471 channel.write("reload", appRoot.c_str(), NULL);
00472 } catch (SystemException &e) {
00473 e.setBriefMessage("Could not write 'reload' command to the spawn server");
00474 throw;
00475 }
00476 }
00477
00478 void handleReloadException(const SystemException &e, const string &appRoot) {
00479 TRACE_POINT();
00480 bool restarted;
00481 try {
00482 P_DEBUG("Spawn server died. Attempting to restart it...");
00483 restartServer();
00484 P_DEBUG("Restart seems to be successful.");
00485 restarted = true;
00486 } catch (const IOException &e) {
00487 P_DEBUG("Restart failed: " << e.what());
00488 restarted = false;
00489 } catch (const SystemException &e) {
00490 P_DEBUG("Restart failed: " << e.what());
00491 restarted = false;
00492 }
00493 if (restarted) {
00494 return sendReloadCommand(appRoot);
00495 } else {
00496 throw SpawnException("The spawn server died unexpectedly, and restarting it failed.");
00497 }
00498 }
00499
00500 IOException prependMessageToException(const IOException &e, const string &message) {
00501 return IOException(message + ": " + e.what());
00502 }
00503
00504 SystemException prependMessageToException(const SystemException &e, const string &message) {
00505 return SystemException(message + ": " + e.brief(), e.code());
00506 }
00507
00508 protected:
00509
00510
00511
00512
00513 virtual void spawnServerStarted() { }
00514
00515 public:
00516
00517
00518
00519
00520
00521
00522
00523
00524
00525
00526
00527
00528
00529
00530
00531
00532 SpawnManager(const string &spawnServerCommand,
00533 const ServerInstanceDir::GenerationPtr &generation,
00534 const AccountsDatabasePtr &accountsDatabase = AccountsDatabasePtr(),
00535 const string &rubyCommand = "ruby",
00536 const AnalyticsLoggerPtr &analyticsLogger = AnalyticsLoggerPtr(),
00537 int logLevel = 0,
00538 const string &debugLogFile = ""
00539 ) {
00540 TRACE_POINT();
00541 this->spawnServerCommand = spawnServerCommand;
00542 this->generation = generation;
00543 this->accountsDatabase = accountsDatabase;
00544 this->rubyCommand = rubyCommand;
00545 this->analyticsLogger = analyticsLogger;
00546 this->logLevel = logLevel;
00547 this->debugLogFile = debugLogFile;
00548 pid = 0;
00549 this_thread::disable_interruption di;
00550 this_thread::disable_syscall_interruption dsi;
00551 try {
00552 restartServer();
00553 } catch (const IOException &e) {
00554 throw prependMessageToException(e, "Could not start the spawn server");
00555 } catch (const SystemException &e) {
00556 throw prependMessageToException(e, "Could not start the spawn server");
00557 }
00558 }
00559
00560 virtual ~SpawnManager() {
00561 TRACE_POINT();
00562 if (pid != 0) {
00563 UPDATE_TRACE_POINT();
00564 this_thread::disable_interruption di;
00565 this_thread::disable_syscall_interruption dsi;
00566 syscalls::unlink(socketFilename.c_str());
00567 ownerSocket.close();
00568 syscalls::waitpid(pid, NULL, 0);
00569 }
00570 }
00571
00572 virtual ProcessPtr spawn(const PoolOptions &options) {
00573 TRACE_POINT();
00574 AnalyticsScopeLog scope(options.log, "spawn app process");
00575 ProcessPtr result;
00576 boost::mutex::scoped_lock l(lock);
00577
00578 try {
00579 result = sendSpawnCommand(options);
00580 } catch (const SpawnException &e) {
00581 if (e.hasErrorPage()) {
00582 throw;
00583 } else {
00584 result = handleSpawnException(e, options);
00585 }
00586 }
00587 scope.success();
00588 return result;
00589 }
00590
00591 virtual void reload(const string &appRoot) {
00592 TRACE_POINT();
00593 this_thread::disable_interruption di;
00594 this_thread::disable_syscall_interruption dsi;
00595 try {
00596 return sendReloadCommand(appRoot);
00597 } catch (const SystemException &e) {
00598 return handleReloadException(e, appRoot);
00599 }
00600 }
00601
00602 virtual void killSpawnServer() const {
00603 kill(pid, SIGKILL);
00604 }
00605
00606 virtual pid_t getServerPid() const {
00607 return pid;
00608 }
00609 };
00610
00611
00612 typedef shared_ptr<SpawnManager> SpawnManagerPtr;
00613
00614 }
00615
00616 #endif