/* * HistoryCommunication.c * mpsLinkProcessor * * Created by Stephen Norum. * Copyright 2008 Stanford Linear Accelerator Center. All rights reserved. * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "MPSFaultNumbers.h" #include "HistoryCommunication.h" #include "Priority.h" #if 0 #pragma mark - #pragma mark Structs, typedefs, Local and Global Variables #endif enum { MPSHistoryServerTimeToLiveReset = 20, MPSHistoryServerTimeToLiveTimerResolutionSeconds = 5, MPSHistoryShouldLogWithAlternateMethod = -1 }; struct historyServerStruct { ELLNODE listNode; unsigned int timeToLive; /* Number of seconds or message attempts remaining before server will be removed from list */ struct sockaddr_in address; /* Network order IP address */ char addressString[INET_ADDRSTRLEN]; /* String representation of IP address */ }; typedef struct historyServerStruct historyServerElement; static ELLLIST historyServerList; static ELLLIST freeHistoryServerList; epicsMessageQueueId MPSHistoryMessageQueue = 0; static epicsThreadId historyMessageSenderThreadId = 0; static int historyClientSocket = 0; static epicsThreadId historyServerNotificationReceiverThreadId = 0; static epicsThreadId historyServerTimeToLiveThreadId = 0; static epicsMutexId historyServerMutex = 0; static historyServerElement *connectedHistoryServer = NULL; static int allowHistoryMessages = epicsTrue; static int showDebug = epicsFalse; static int historyProtocolVersion = MPSHistoryProtocolVersion1; static int connectToHistoryServer(historyServerElement *hse); static int disconnectFromHistoryServer(); #if 0 #pragma mark - #pragma mark Main Thread #endif static int logMessageWithAlternateMethod(MPSHistoryMessage *historyMessage) { errlogPrintf("MPSLOG: type: %u at: %u.%09u:%u id: %u val: %u -> %u\n", historyMessage->type, historyMessage->timestampWithPulseId.secPastEpoch, historyMessage->timestampWithPulseId.nsec, historyMessage->timeslot, historyMessage->id, historyMessage->oldValue, historyMessage->newValue); return MPS_SUCCESS; } static int sendMessage(const void *buffer, size_t length) { int i; int bytesSent = 0; epicsMutexLock(historyServerMutex); if (historyClientSocket == 0) { bytesSent = MPSHistoryShouldLogWithAlternateMethod; } else { for (i = 0; i < MPSHistoryMaxRetryCount; i++) { bytesSent = send(historyClientSocket, buffer, length, 0x00); if (bytesSent == length) { /* Success */ if (connectedHistoryServer) { connectedHistoryServer->timeToLive = MPSHistoryServerTimeToLiveReset; } break; } } } epicsMutexUnlock(historyServerMutex); return bytesSent; } static unsigned int prepareSuperMessageVersion1(MPSHistoryMessage *historyMessage, char *buffer, unsigned int bufferSize) { char *header; char *data; if (bufferSize < HISTORY_SUPER_MESSAGE_SIZE_V1) { return 0; } header = HISTORY_MESSAGE_HEADER_SECTION_POINTER(buffer); data = HISTORY_MESSAGE_DATA_SECTION_POINTER(buffer); /* Header */ HISTORY_MESSAGE_PROTOCOL_VERSION(header) = MPSHistoryProtocolVersion1; HISTORY_MESSAGE_TYPE(header) = MPSHistoryMessageTypeSuper; /* Data */ HISTORY_SUPER_TIMESTAMP_SECPASTEPOCH_V1(data) = htonl(historyMessage->timestampWithPulseId.secPastEpoch); HISTORY_SUPER_TIMESTAMP_NSEC_PULSEID_V1(data) = htonl(historyMessage->timestampWithPulseId.nsec); HISTORY_SUPER_ID_V1(data) = htonl(historyMessage->id); HISTORY_SUPER_TIMESLOT_V1(data) = historyMessage->timeslot; HISTORY_SUPER_TYPE_V1(data) = historyMessage->type; HISTORY_SUPER_OLD_VALUE_V1(data) = (epicsUInt8) historyMessage->oldValue; HISTORY_SUPER_NEW_VALUE_V1(data) = (epicsUInt8) historyMessage->newValue; return HISTORY_SUPER_MESSAGE_SIZE_V1; } static unsigned int prepareSuperMessageVersion2(MPSHistoryMessage *historyMessage, char *buffer, unsigned int bufferSize) { char *header; char *data; if (bufferSize < HISTORY_SUPER_MESSAGE_SIZE_V2) { return 0; } header = HISTORY_MESSAGE_HEADER_SECTION_POINTER(buffer); data = HISTORY_MESSAGE_DATA_SECTION_POINTER(buffer); /* Header */ HISTORY_MESSAGE_PROTOCOL_VERSION(header) = MPSHistoryProtocolVersion2; HISTORY_MESSAGE_TYPE(header) = MPSHistoryMessageTypeSuper; /* Data */ HISTORY_SUPER_TIMESTAMP_SECPASTEPOCH_V2(data) = htonl(historyMessage->timestampWithPulseId.secPastEpoch); HISTORY_SUPER_TIMESTAMP_NSEC_PULSEID_V2(data) = htonl(historyMessage->timestampWithPulseId.nsec); HISTORY_SUPER_ID_V2(data) = htonl(historyMessage->id); HISTORY_SUPER_TIMESLOT_V2(data) = historyMessage->timeslot; HISTORY_SUPER_TYPE_V2(data) = historyMessage->type; HISTORY_SUPER_OLD_VALUE_V2(data) = htonl(historyMessage->oldValue); HISTORY_SUPER_NEW_VALUE_V2(data) = htonl(historyMessage->newValue); return HISTORY_SUPER_MESSAGE_SIZE_V2; } static void historyMessageSenderThread(void *arg) { int err; MPSHistoryMessage historyMessage; char buffer[512]; unsigned int messageLength; int protocolVersion; while(1) { err = epicsMessageQueueReceive(MPSHistoryMessageQueue, &historyMessage, sizeof(MPSHistoryMessage)); if (err <= 0) { continue; } epicsMutexLock(historyServerMutex); protocolVersion = historyProtocolVersion; epicsMutexUnlock(historyServerMutex); switch(protocolVersion) { case MPSHistoryProtocolVersion1: messageLength = prepareSuperMessageVersion1(&historyMessage, buffer, sizeof(buffer)); break; case MPSHistoryProtocolVersion2: messageLength = prepareSuperMessageVersion2(&historyMessage, buffer, sizeof(buffer)); break; default: messageLength = 0; break; } if (messageLength == 0) { continue; } /* Send the message */ err = sendMessage(buffer, messageLength); if (err == MPSHistoryShouldLogWithAlternateMethod) { logMessageWithAlternateMethod(&historyMessage); continue; } if (err != messageLength) { /* History message send failed */ errlogPrintf("historyMessageSenderThread: Couldn't send message. Sent %d of %d bytes.\n", err, messageLength); logMessageWithAlternateMethod(&historyMessage); } } cantProceed("Left historyMessageSenderThread infinite loop"); } int mpsHistorySendTestMessages() { static int faultNumber = 0; int i; MPSHistoryMessage historyMessage; historyMessage.type = MPSHistoryMessageTypeFaultState; historyMessage.timestampWithPulseId.secPastEpoch = 0xABCDABCD; historyMessage.timestampWithPulseId.nsec = 0x12345678; historyMessage.timeslot = 0xFF; historyMessage.oldValue = epicsTrue; historyMessage.newValue = epicsFalse; for (i = 1; i <= FAULT_COUNT; i++) { historyMessage.id = faultNumber++; epicsMessageQueueSend(MPSHistoryMessageQueue, &historyMessage, sizeof(historyMessage)); } return MPS_SUCCESS; } int mpsHistoryAddMessageToQueue(MPSHistoryMessage *historyMessage, int messageLength) { if (allowHistoryMessages == epicsFalse) { return MPS_SUCCESS; } if (epicsMessageQueueTrySend(MPSHistoryMessageQueue, historyMessage, messageLength) != 0 ) { logMessageWithAlternateMethod(historyMessage); } return MPS_SUCCESS; } #if 0 #pragma mark - #pragma mark History Server Notification #endif void printHistoryServerList() { historyServerElement *hse = NULL; epicsMutexLock(historyServerMutex); hse = (historyServerElement *) ellFirst(&historyServerList); while(hse != NULL) { printf("%s:%u\tTime-to-live: %d\tIs Connected: %s\n", hse->addressString, ntohs(hse->address.sin_port), hse->timeToLive, hse == connectedHistoryServer ? "Yes" : "No"); hse = (historyServerElement *) ellNext((ELLNODE *) hse); } epicsMutexUnlock(historyServerMutex); } /* Lock historyServerMutex before calling */ static historyServerElement * findServerInHistoryServerListByAddress (struct in_addr in) { historyServerElement *hse; hse = (historyServerElement *) ellFirst(&historyServerList); while (hse != NULL) { if (hse->address.sin_addr.s_addr == in.s_addr) { return hse; } hse = (historyServerElement *) ellNext((ELLNODE *) hse); } return NULL; } /* Lock historyServerMutex before calling */ static int connectToHistoryServer(historyServerElement *hse) { int err = MPS_ERROR; int newSocket = 0; int optionValue = 0; if (showDebug) { printf("connectToHistoryServer: attempting to connect to %s:%d\n", hse->addressString, ntohs(hse->address.sin_port)); } if (hse == NULL) { return MPS_ERROR_ARGUMENT_OUT_OF_RANGE; } disconnectFromHistoryServer(); newSocket = socket(AF_INET, SOCK_STREAM, 0); if (newSocket == -1) { return MPS_ERROR_SOCKET_CREATION_FAILED; } /* Set keep-alive socket option */ optionValue = 1; setsockopt(newSocket, SOL_SOCKET, SO_KEEPALIVE, &optionValue, sizeof(optionValue)); err = connect(newSocket, (struct sockaddr *) &(hse->address), sizeof(hse->address)); if (err < 0) { perror("connectToHistoryServer"); return MPS_ERROR; } if (showDebug) { printf("connectToHistoryServer: connected to %s:%d\n", hse->addressString, ntohs(hse->address.sin_port)); } hse->timeToLive = MPSHistoryServerTimeToLiveReset; connectedHistoryServer = hse; historyClientSocket = newSocket; return MPS_SUCCESS; } /* Lock historyServerMutex before calling */ static int disconnectFromHistoryServer() { if (historyClientSocket > 0) { close(historyClientSocket); } historyClientSocket = 0; connectedHistoryServer = NULL; return MPS_SUCCESS; } static void historyServerNotificationReceiverThread(void *arg) { int err; int notificationSocket; fd_set readSet; struct sockaddr_in notificationSource; struct sockaddr_in sender; socklen_t senderLength; static char buffer[NOTIFICATION_MESSAGE_SIZE]; ssize_t bytesReceived; char addressString[INET_ADDRSTRLEN]; epicsUInt8 protocolVersion; char *header; char *data; historyServerElement *hse; int needsReconnect; /* Create socket to listen for notifications */ notificationSocket = socket(AF_INET, SOCK_DGRAM, 0); if (notificationSocket < 0) { cantProceed("historyServerNotificationReceiverThread: could not create socket\n"); } memset((char *) ¬ificationSource, 0x0, sizeof(notificationSource)); notificationSource.sin_family = AF_INET; notificationSource.sin_addr.s_addr = htonl(INADDR_ANY); notificationSource.sin_port = htons(MPSHistoryProtocolNotificationPort); err = bind(notificationSocket, (struct sockaddr *) ¬ificationSource, sizeof(notificationSource)); if (err < 0) { fprintf(stderr, "historyServerNotificationReceiverThread: could not bind\n"); } while(1) { FD_ZERO(&readSet); FD_SET(notificationSocket, &readSet); /* Wait for data to arrive */ if (select(notificationSocket+1, &readSet, NULL, NULL, NULL)) { /* Receive the data */ senderLength = sizeof(sender); bytesReceived = recvfrom(notificationSocket, buffer, sizeof(buffer)/sizeof(buffer[0]), 0, (struct sockaddr *) &sender, &senderLength); if (bytesReceived < 0) { printf("historyServerNotificationReceiverThread: received bad history server notification"); continue; } header = HISTORY_MESSAGE_HEADER_SECTION_POINTER(buffer); data = HISTORY_MESSAGE_DATA_SECTION_POINTER(header); protocolVersion = HISTORY_MESSAGE_PROTOCOL_VERSION(header); /* Check the message version */ if ((protocolVersion != MPSHistoryProtocolVersion1) && (protocolVersion != MPSHistoryProtocolVersion2)) { inet_ntop(AF_INET, &(sender.sin_addr), addressString, sizeof(addressString)/sizeof(addressString[0])); errlogPrintf("historyServerNotificationReceiverThread: received out-of-date history server notification from %s\n", addressString); continue; } /* Check the message type */ if (HISTORY_MESSAGE_TYPE(header) != MPSHistoryMessageTypeNotification) { inet_ntop(AF_INET, &(sender.sin_addr), addressString, sizeof(addressString)/sizeof(addressString[0])); errlogPrintf("historyServerNotificationReceiverThread: received unexpected MPS history message type from %s\n", addressString); continue; } needsReconnect = epicsFalse; epicsMutexLock(historyServerMutex); hse = findServerInHistoryServerListByAddress(sender.sin_addr); if (hse == NULL) { /* Server is not known, grab a free node from freeHistoryServerList */ hse = (historyServerElement *) ellGet(&freeHistoryServerList); if (hse == NULL) { /* No free nodes are available */ epicsMutexUnlock(historyServerMutex); errlogPrintf("historyServerNotificationReceiverThread no free nodes in freeHistoryServerList. Server not added."); continue; } else { ellAdd(&historyServerList, (ELLNODE *) hse); memset((char *) &(hse->address), 0x0, sizeof(hse->address)); hse->address.sin_family = sender.sin_family; hse->address.sin_addr.s_addr = sender.sin_addr.s_addr; hse->address.sin_port = 0; inet_ntop(AF_INET, &(hse->address.sin_addr), hse->addressString, sizeof(hse->addressString)/sizeof(hse->addressString[0])); } } if (hse->address.sin_port != NOTIFICATION_SERVER_PORT_NUMBER(data)) { hse->address.sin_port = NOTIFICATION_SERVER_PORT_NUMBER(data); needsReconnect = epicsTrue; } hse->timeToLive = MPSHistoryServerTimeToLiveReset; if ((connectedHistoryServer == hse) && (needsReconnect == epicsTrue)) { connectToHistoryServer(hse); } if (connectedHistoryServer == NULL) { connectToHistoryServer(hse); } epicsMutexUnlock(historyServerMutex); } else { /* Reaches here if timeout is included in select call */ fprintf(stderr, "historyServerNotificationReceiverThread: Warning, no history servers found.\n"); } } cantProceed("Left historyServerNotificationReceiverThread infinite loop"); } /* Lock historyServerMutex before calling */ static void removeHistoryServerFromList(historyServerElement *hse) { /* Return the node to the free list */ ellDelete(&historyServerList, (ELLNODE *) hse); ellAdd (&freeHistoryServerList, (ELLNODE *) hse); /* If hse is the connected history server, attempt to use a new server */ if (hse == connectedHistoryServer) { /* Disconnect from this server */ disconnectFromHistoryServer(); hse = (historyServerElement *) ellFirst(&historyServerList); if (hse != NULL) { /* There is another active server. Try to connect to it. */ connectToHistoryServer(hse); } } } static void updateHistoryServersTimeToLive(void *arg) { historyServerElement *hse; unsigned int timeSinceLastCall; int needsNewConnection; while (1) { timeSinceLastCall = MPSHistoryServerTimeToLiveTimerResolutionSeconds; needsNewConnection = epicsFalse; epicsMutexLock(historyServerMutex); hse = (historyServerElement *) ellFirst(&historyServerList); while (hse != NULL) { if (hse->timeToLive > timeSinceLastCall) { /* The server's time to live has not expired. Update the remaining time. */ hse->timeToLive -= timeSinceLastCall; } else { /* The server's time to live has expired */ hse->timeToLive = 0; } if (hse->timeToLive == 0) { removeHistoryServerFromList(hse); } /* Check the next server */ hse = (historyServerElement *) ellNext((ELLNODE *) hse); } epicsMutexUnlock(historyServerMutex); sleep(MPSHistoryServerTimeToLiveTimerResolutionSeconds); } cantProceed("Left updateHistoryServersTimeToLive infinite loop"); } #if 0 #pragma mark - #pragma mark Initialization #endif void setAllowHistoryMessages(int state) { allowHistoryMessages = state ? epicsTrue : epicsFalse; } void mpsSetHistoryProtocolVersion(int version) { epicsMutexLock(historyServerMutex); switch (version) { case MPSHistoryProtocolVersion1: historyProtocolVersion = MPSHistoryProtocolVersion1; break; case MPSHistoryProtocolVersion2: historyProtocolVersion = MPSHistoryProtocolVersion2; break; default: historyProtocolVersion = MPSHistoryProtocolVersion2; break; } epicsMutexUnlock(historyServerMutex); } int initializeHistoryCommunication() { int i; historyServerElement *hse; historyServerMutex = epicsMutexMustCreate(); /* * Create historyServerElements for a few servers */ /* Get memory */ hse = (historyServerElement *) callocMustSucceed(MPSHistoryServerListCapacity, sizeof(historyServerElement), "initializeHistoryCommunication"); /* Initialize bypass lists */ epicsMutexLock(historyServerMutex); ellInit(&historyServerList); ellInit(&freeHistoryServerList); /* Add the nodes to the free list */ for (i = 0; i < MPSHistoryServerListCapacity; i++) { /* Add the node to the free list */ ellAdd(&freeHistoryServerList, (ELLNODE *) hse); hse++; } epicsMutexUnlock(historyServerMutex); /* Allow sending of history messages version 1 */ setAllowHistoryMessages(epicsTrue); mpsSetHistoryProtocolVersion(MPSHistoryProtocolVersion2); /* Create message queue for fault data */ MPSHistoryMessageQueue = epicsMessageQueueCreate(MPSHistoryMessageQueueCapacity, sizeof(MPSHistoryMessage)); /* Start a thread to take messages from the queue and send them to the history server */ historyMessageSenderThreadId = epicsThreadCreate("mpsHistoryMessageSenderThread", MPSThreadPriorityLow, epicsThreadGetStackSize(epicsThreadStackMedium), (EPICSTHREADFUNC) historyMessageSenderThread, NULL); /* Start a thread to keep track of history servers' times-to-live */ historyServerTimeToLiveThreadId = epicsThreadCreate("mpsUpdateHistoryServersTimeToLive", MPSThreadPriorityLow, epicsThreadGetStackSize(epicsThreadStackSmall), (EPICSTHREADFUNC) updateHistoryServersTimeToLive, NULL); /* Start a thread to take receive notifications from history servers */ historyServerNotificationReceiverThreadId = epicsThreadCreate("mpsHistoryServerNotificationReceiverThread", MPSThreadPriorityLow, epicsThreadGetStackSize(epicsThreadStackSmall), (EPICSTHREADFUNC) historyServerNotificationReceiverThread, NULL); return MPS_SUCCESS; }