class WalRecTestCase(MPPTestCase):
    ''' Testcases for SIGTERM on wal receiver'''

    def __init__(self, methodName):
        self.gp = GpinitStandby()
        self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
        self.pgutil = GpUtility()
        super(WalRecTestCase,self).__init__(methodName)

    def setUp(self):
        #Remove standby if present
        self.gp.run(option='-r')
       
    def invoke_sigterm_and_verify(self):
        ''' Invoke sigterm on wal receiver and verify that a new process is spawned after '''
        gpact_stdby = GpactivateStandby()
        standby_host = gpact_stdby.get_current_standby()
        standby_port = gpact_stdby.get_standby_port()
        wal_rec_pid_1 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
        sig_cmd = "gpssh -h %s -e 'kill -15 %s'" % (standby_host, wal_rec_pid_1)
        cmd = Command('Issue SIGTERM to wam receiver process', cmdStr=sig_cmd)
        tinctest.logger.info ('%s' % cmd)
        cmd.run(validateAfter=True)
        result = cmd.get_results()
        if result.rc != 0:
            return False        
        wal_rec_pid_2 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
        if wal_rec_pid_1 == wal_rec_pid_2:
            return False
        return True

    @unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
    def test_sigterm_on_walreceiver(self):
        self.pgutil.install_standby()
        self.assertTrue(self.invoke_sigterm_and_verify())

 def test_run_five(self):
     for i in xrange(5):
         with NewEnv(PGPORT=self.port,
                     MASTER_DATA_DIRECTORY=self.mdd):        
             pguti = GpUtility() 
             if i == 0:
                 pguti.install_standby(socket.gethostname(), self.mdd)
             # starting from second time, init standby from new master, standby_dir will be like  master_newstandby_newstandby...
             else:
                 pguti.install_standby(socket.gethostname(), os.path.join(self.mdd,'newstandby'))
             gpact = GpactivateStandby()
             self.mdd = gpact.get_standby_dd()
             self.port = gpact.get_standby_port()
             gpact.activate()
             tinctest.logger.info("self.mdd is %s, self.port is %s"%(self.mdd, self.port))

 def __init__(self):
     self.stdby = StandbyVerify()
     self.runmixin = StandbyRunMixin()
     self.runmixin.createdb(dbname='walrepl')        
     self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
     self.config = GPDBConfig()
     self.pgutil = GpUtility()
     self.host = socket.gethostname()

 def __init__(self):
     self.gpinit = GpinitStandby()
     self.mdd = os.environ.get("MASTER_DATA_DIRECTORY")
     self.config = GPDBConfig()
     self.pgutil = GpUtility()
     self.verify = StandbyVerify()
     self.host = socket.gethostname()
     self.standby_loc = os.path.join(os.path.split(self.mdd)[0], self.standby_dirname)
     self.standby = Standby(self.standby_loc, self.standby_port)

class OOMGpstartTestCase(MPPTestCase):
    ''' Simulating oom on standby master , gpstart fails to start standby'''
    
    # Setup OOM env
    oom.setup_oom()

    def __init__(self, methodName):
        self.gpinitstandby= GpinitStandby()
        self.gputil = GpUtility()
        super(OOMGpstartTestCase,self).__init__(methodName)

    def setUp(self):
        self.gpinitstandby.run(option='-r')

    def tearDown(self):
        self.gpinitstandby.run(option='-r')

    def test_oom_with_gpstart(self):
        # Create standby
        oom.create_standby()
        # Stop cluster
        self.gputil.run('gpstop -a')
        # Run gpstart after creaitng the flag file- Presence of this flag file will return NULL for malloc simulating a no  memory scenario
        oom.touch_malloc()
        self.assertFalse(oom.startdb(), 'Standby start is expected to fail with no memory error')
        # Restart after removing the flag file
        self.assertTrue(oom.restartdb(), 'Expected a successful start of standby')

    @unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
    def test_oom_with_psql_con(self):
        # Create standby
        oom.create_standby()
        # Stop cluster
        self.gputil.run('gpstop -a')
        # Run gpstart with wrapper
        oom.startdb()
        # OOM with psql- after adding the flag file to the standby directory to simulate OOM
        self.assertTrue(oom.psql_and_oom())
        # Start standby after free up memory
        self.assertTrue(oom.start_standby())

 def init_standby(self):
     pg = GpUtility() 
     pg.install_standby()

class PromoteTestCase(MPPTestCase):
   '''testcase for gpstart''' 
 
   def __init__(self,methodName):
       self.pgutil = GpUtility()
       super(PromoteTestCase,self).__init__(methodName)
  
   def setUp(self):
       self.pgutil.check_and_start_gpdb()
       # We should forcibly recreate standby, as it might has been promoted.
       self.pgutil.remove_standby()
       self.pgutil.install_standby()


   def tearDown(self):
       self.pgutil.remove_standby()

   def test_promote_incomplete_stdby(self):
       ''' 
       remove the standby base dir, try promote and check if fail       
       '''
       gpactivate_stdby = GpactivateStandby()
       gpinit_stdby = GpinitStandby()
       stdby_mdd = gpactivate_stdby.get_standby_dd()
       stdby_host = gpinit_stdby.get_standbyhost()
       stdby_port = gpactivate_stdby.get_standby_port()
       destDir = os.path.join(stdby_mdd, 'base')
       self.pgutil.clean_dir(stdby_host,destDir)
       promote_cmd = "pg_ctl promote -D %s"%stdby_mdd       
       (rc, output) = gpactivate_stdby.run_remote(stdby_host,promote_cmd ,stdby_port,stdby_mdd)
       self.assertEqual(rc, 0)
       pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword='master', option='bin')
       self.assertTrue(int(pid) == -1, 'incomplete standby data directory promote succeeds.')

from mpp.models import MPPTestCase
from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl.gpactivatestandby import GpactivateStandby
from mpp.gpdb.tests.storage.walrepl.gpinitstandby import GpinitStandby
from mpp.gpdb.tests.storage.walrepl.lib import NewEnv
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.PSQL import PSQL
from tinctest.lib import local_path
import tinctest

import os
import shutil
import socket
from time import sleep

pgutil = GpUtility()
pgutil.check_and_start_gpdb()
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')

class TakeFive(MPPTestCase):

    port = os.environ.get('PGPORT')
    mdd = os.environ.get('MASTER_DATA_DIRECTORY')

    def __init__(self, methodName):
        super(TakeFive, self).__init__(methodName)
   
    def setUp(self):
        Command('gpinitstandby -ra', 'gpinitstandby -ra').run()

    def tearDown(self):

limitations under the License.
"""

import unittest2 as unittest
from tinctest import logger

from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl import lib as walrepl
from mpp.gpdb.tests.storage.walrepl.walreceiver import GPLibPQTestCase
from mpp.gpdb.tests.storage.walrepl.lib.pqwrap import *
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.config import GPDBConfig

import select

gputil = GpUtility()
gputil.check_and_start_gpdb()
config = GPDBConfig()

class case(GPLibPQTestCase):
    """Basic test cases to see gplibpq and walsender are talking correctly."""

    def setUp(self):
        # cleanup
        cmd = Command('gpinitstandby', 'gpinitstandby -ar')
        # don't care the result in case standby is not configured
        cmd.run()
    
    @unittest.skipIf(config.is_multinode(), "Test applies only to a singlenode cluster")
    def test_connect(self):
        """

 def __init__(self,methodName):
     self.gputil = GpUtility()
     super(GpstopTestCase,self).__init__(methodName)

class GpstopTestCase(MPPTestCase):
   '''testcase for gpstart''' 
    
   origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY') 
   
   def __init__(self,methodName):
       self.gputil = GpUtility()
       super(GpstopTestCase,self).__init__(methodName)
    
    
   def setUp(self):
       self.gputil.check_and_start_gpdb()     
       stdby_presence = self.gputil.check_standby_presence()
       if  stdby_presence:
           self.gputil.remove_standby()
       self.gputil.install_standby()

   def tearDown(self):
       self.gputil.remove_standby()
       self.gputil.run('gpstart -a')
       self.gputil.run('gprecoverseg -a')

   def test_gpstop_from_master(self):
       self.assertTrue(self.gputil.gpstop_and_verify())
       self.gputil.run('gpstart -a')

   def test_gpstop_master_only(self):
       self.assertTrue(self.gputil.gpstop_and_verify(option = '-m'))
       self.gputil.run('gpstart -a')

   def test_gpstop_fast(self):
       #run transactions, and stop fast, check if transaction aborted, and the cluster was stopped
       self.assertTrue(self.gputil.gpstop_and_verify(option = '-M fast'))
       self.gputil.run('gpstart -a')

   def test_gpstop_immediate(self):
       self.assertTrue(self.gputil.gpstop_and_verify(option = '-M immediate'))
       self.gputil.run('gpstart -a')

   def test_gpstop_smart(self):
       self.assertTrue(self.gputil.gpstop_and_verify(option = '-M smart'))
       self.gputil.run('gpstart -a')
  
   def test_gpdb_restart(self):
       self.assertTrue(self.gputil.gpstop_and_verify('-r'))

   def test_gpdb_reload(self):
       self.assertTrue(self.gputil.gpstop_and_verify('-u'))

   def test_gpstop_except_stdby(self):
       self.assertTrue(self.gputil.gpstop_and_verify('-y'))
       self.gputil.run('gpstart -y')

   def test_gpstop_after_failover(self):
       tinctest.logger.info("test gpstop from new master after failover")
       activatestdby = GpactivateStandby()
       standby_host = activatestdby.get_current_standby()
       standby_port =  activatestdby.get_standby_port()
       standby_mdd = activatestdby.get_standby_dd()
       activatestdby.activate()    
       (rc,stdout)=activatestdby.run_remote(standby_host, rmt_cmd='gpstop -a -M fast',
                                                 pgport=standby_port,standbydd=standby_mdd) 
       self.assertEqual(0,rc)
       activatestdby.run_remote(standby_host, rmt_cmd='gpstart -a',
                                                 pgport=standby_port,
                                                 standbydd=standby_mdd)
       self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port) 

    def test_tli_mismatch(self):
        """
        Test to verify if TLI mismatch issue during Pass 3 of xlog record
        (checkpoint) replay occurs or not. A set of checkpoints one after
        the other when replayed on the standby and then if the 
        standby is promoted, it should go through the prmotion just fine.

        The flow of this test is as follows.
        1. Initiate the Standby using the Master (primary) postmaster
           paramerters.
        2. Perform explicit checkpoints and wait so that they get replicated.
        3. Then promote the standby, wait and then try to access it.
        4. If we can successfuly access it, its a Pass otherwise a Fail.
        """

        PSQL.run_sql_command('DROP table if exists foo')
        PSQL.run_sql_command('create table foo (a int)')
        PSQL.run_sql_command('insert into foo '
                             'select * from generate_series(1, 1000)')

        # Initial setup, forcibly remove standby and install new one
        pgutil = GpUtility()
        pgutil.remove_standby()   
        logger.info ('\nCreate a standby...')
        res = self.standby.create()
        self.assertEqual(res, 0)
        res = self.standby.start()
        self.assertTrue(res.wasSuccessful())

        # Wait for the walreceiver to start
        num_walsender = self.wait_for_walsender()
        self.assertEqual(num_walsender, 1)

        logger.info('Standby activated...')
        logger.info('Give the standby some time to catchup...')
        time.sleep(3)

        logger.info('Create checkpoints and let them get replicated...')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')
        PSQL.run_sql_command('checkpoint')

        time.sleep(2)

        # Promote standby
        # We don't kill/stop the primary, as it's convenient for
        # next testing
        logger.info('Promote the standby immediatly')
        self.standby.promote()

        logger.info('Wait for the standby to be ready to accept connections ...')
        time.sleep(3)

        # Verify the result replicated to the standby.
        logger.info('Verify if table foo exists...')
        proc = self.run_sql('select count(*) from foo',
                            str(self.standby.port))

        # The table should exist
        stdout = proc.communicate()[0]
        logger.info(stdout)
        search = "1000"
        self.assertTrue(stdout.find(search) >= 0)

        logger.info('Pass')

 def __init__(self, methodName):
     self.gpinitstandby= GpinitStandby()
     self.gputil = GpUtility()
     super(OOMGpstartTestCase,self).__init__(methodName)

class GpinitStandby(object):
    '''Class for gpinitstandby operations 
       Disclaimer: Some of these may repeat with the mpp/lib version'''
    def __init__(self):
        self.stdby = StandbyVerify()
        self.runmixin = StandbyRunMixin()
        self.runmixin.createdb(dbname='walrepl')        
        self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
        self.config = GPDBConfig()
        self.pgutil = GpUtility()
        self.host = socket.gethostname()
 
    def run(self, option = ''):
        '''Runs gpinitstandby and returns True if successfull '''
        gpinitstandby_cmd = 'gpinitstandby -a %s' % option
        cmd = Command(name='Running Gpinitstandby', cmdStr="%s" % gpinitstandby_cmd)
        tinctest.logger.info(" %s" % cmd)
        cmd.run(validateAfter=False)
        result = cmd.get_results()
        if result.rc != 0:
            return False
        return True

    def verify_gpinitstandby(self, primary_pid):  
        '''Verify the presence of standby in recovery mode '''
        if (self.stdby.check_gp_segment_config()) and (self.stdby.check_pg_stat_replication()) and (self.stdby.check_standby_processes())and self.compare_primary_pid(primary_pid) :
            return True
        return False

    def get_masterhost(self):
        std_sql = "select hostname from gp_segment_configuration where content=-1 and role='p';"
        master_host = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'postgres')
        return master_host.strip()

    def get_standbyhost(self):
        std_sql = "select hostname from gp_segment_configuration where content='-1' and role='m';"
        standby_host = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'postgres')
        return standby_host.strip()

    def get_filespace_location(self):
        fs_sql = "select fselocation from pg_filespace_entry where fselocation like '%fs_walrepl_a%' and fsedbid=1;"
        filespace_loc = PSQL.run_sql_command(fs_sql, flags = '-q -t', dbname= 'postgres')
        return filespace_loc.strip()

    def get_standbyhostnode(self):
        '''
        Function used to obtain the hostname of one of the segment node inorder to use it as the standby master node" 
        @return : returns the hostname of the segment node which can be used as the standby master node
        '''
        hostlist = self.config.get_hosts()
        standby = ''
        for host in hostlist:
            if host.strip() != self.host:
                standby = host.strip()
        if len(standby) > 0 :
            return standby
        else:
            tinctest.logger.error('No segment host other than master available to have remote standby')

    def get_primary_pid(self):
        pid = self.pgutil.get_pid_by_keyword(pgport=os.environ.get('PGPORT'), keyword=self.mdd)
        if int(pid) == -1:
            raise WalReplException('Unable to get pid of primary master process')
        else:
            return int(pid)

    def compare_primary_pid(self, initial_pid):
        final_pid = self.get_primary_pid()
        if initial_pid == final_pid :
            return True
        return False

    def create_dir_on_standby(self, standby, location):
        fs_cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s' " % (standby, location, location)
        cmd = Command(name='Make dierctory on standby before running the command', cmdStr = fs_cmd)
        tinctest.logger.info('%s' % cmd)
        cmd.run(validateAfter=True)
        result = cmd.get_results()
        if result.rc != 0:
            raise WalReplException('Unable to create directory on standby')
        else:
            return True
      
    def initstand_by_with_default(self):
        master_host = self.get_masterhost()
        gp_cmd =  "/bin/bash -c 'gpinitstandby -s %s'" % (master_host)
        logfile = open(local_path('install.log'),'w')

        child = pexpect.spawn(gp_cmd, timeout=400)
        child.logfile = logfile
        sleep(2)
        check = child.expect(['.* Enter standby filespace location for filespace pg_system .*', ' '])
        if check != 0:
            child.close()

        l_file = open(local_path('install.log'),'r')
        lines = l_file.readlines()
        for line in lines:
            if 'default: NA' in line:
                return True
#.........这里部分代码省略.........

class OOMClass(object):
    '''Class for methods required for OOM testcase'''

    standby_port = '5433'
    standby_dirname = 'newstandby'

    def __init__(self):
        self.gpinit = GpinitStandby()
        self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
        self.config = GPDBConfig()
        self.pgutil = GpUtility()
        self.verify = StandbyVerify()
        self.host = socket.gethostname()
        self.standby_loc = os.path.join(os.path.split(self.mdd)[0],
                                        self.standby_dirname)
        self.standby = Standby(self.standby_loc, self.standby_port)

    def create_standby(self):
        self.pgutil.clean_dir(self.host,self.standby_loc)
        self.gpinit.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))

    def setup_oom(self):
        # Build it before testing.
        thisdir = os.path.dirname(__file__)
        builddir = os.path.join(thisdir, 'lib')
        subprocess.check_call(['make', '-C', builddir, 'install'])

        #Copy oom_malloc.so and wrapper.sh to all the segment nodes
        for host in config.get_hosts() :
            if host.strip() == self.host :
                continue
            cmd = "gpssh -h %s -e 'mkdir -p %s'; scp %s/oom_malloc.so %s:%s/; scp %s/wrapper.sh %s:%s/" % (host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir)
            self.pgutil.run(cmd)

    def touch_malloc(self):
        # Touch file oom_malloc in standby directory
        cmd = 'touch %s/oom_malloc' % self.standby_loc
        self.pgutil.run(cmd)

    def startdb(self):
        (rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh')))
        if rc != 0 and 'Could not start standby master' in result :
            return False
        return True

    def restartdb(self):
        # Remove file oom_malloc from standby
        cmd = 'rm %s/oom_malloc' % self.standby_loc
        self.pgutil.run(cmd)
        (rc, result) = self.pgutil.run('gpstop -ar')
        if rc == 0 and (self.verify.check_pg_stat_replication()):
            return True
        return False

    def psql_and_oom(self):
        #Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
        self.touch_malloc()
        PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);')
        sleep(2)
        if not (self.verify.check_standby_processes()):
            return True
        return False 

    def start_standby(self):
        # Remove oom_malloc and start standby : Check if all processes are back
        cmd = 'rm %s/oom_malloc' % self.standby_loc
        self.pgutil.run(cmd)
        res = self.standby.start()
        sleep(2)
        if (self.verify.check_standby_processes()) :
            return True
        return False

class WalReplKillProcessTestCase(TINCTestCase):
  
   # this is not hard code, will be updated  
   stdby_host = 'localhost'
   stdby_port = '5432'
 
   def __init__(self,methodName):
       self.gphome = os.environ.get('GPHOME')
       self.pgport = os.environ.get('PGPORT')
       self.pgdatabase = os.environ.get('PGDATABASE')
       self.stdby_host = 'localhost'
       self.master_dd = os.environ.get('MASTER_DATA_DIRECTORY')
       self.pgutil = GpUtility()
       self.stdby = StandbyVerify()
       super(WalReplKillProcessTestCase,self).__init__(methodName)


   def killProcess_byPid(self, signal=9, pid_toKill=[], host="localhost"):
       pid_list = ""
       for pid in pid_toKill:
           pid_list = pid_list + " " + str(pid)

       kill_cmd = "%s/bin/gpssh -h %s -e 'kill -%s  %s'" % (os.environ.get('GPHOME'), host, signal, pid_list)
       (rc, result) = self.pgutil.run(kill_cmd)
       if rc == 0:
           tinctest.logger.info("Process killed, %s" % result)
           return True
       else:
           tinctest.logger.error("Killing process error, Status Code non zero, cmd: %s\n"%kill_cmd)
           return False

   def kill_walstartup(self):
       gpstdby = GpinitStandby()
       stdby_host = gpstdby.get_standbyhost()
       activate_stdby = GpactivateStandby()
       stdby_port = activate_stdby.get_standby_port()
       pid_list = []
       startup_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="startup process")
       if int(startup_pid) == -1:
           tinctest.logger.error("error:startup process does not exist!")
           return False
       else:
           pid_list.append(startup_pid)
           self.killProcess_byPid(pid_toKill=pid_list, host=stdby_host)

   def kill_walreceiver(self):
       gpstdby = GpinitStandby()
       stdby_host = gpstdby.get_standbyhost()
       activate_stdby = GpactivateStandby()
       stdby_port = activate_stdby.get_standby_port()
       pid_list = []
       walreceiver_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="wal receiver process")
       if int(walreceiver_pid) == -1:
           tinctest.logger.error("error: wal receiver process does not exist!")
           return False
       else:  
           pid_list.append(walreceiver_pid)
           self.killProcess_byPid(pid_toKill=pid_list, host=stdby_host)


   def kill_walsender_check_postmaster_reset(self):
       pid_list = []
       walsender_old_pid=self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword="wal sender process")
       if int(walsender_old_pid) == -1:
           tinctest.logger.error("error: process wal sender does not exist on host")
           return False
       else:
           pid_list.append(walsender_old_pid)
           self.killProcess_byPid(pid_toKill=pid_list)
       sleep(2)
       walsender_new_pid=self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword="wal sender process")
       if walsender_old_pid == walsender_new_pid:
          raise Exception("Killing walsender failed to force postmaster reset")
       else:
          return True         

   def kill_transc_backend_check_reset(self):
       dict_process = { 'stats collector process': -1, 'writer process': -1,
                       'checkpointer process': -1,'seqserver process': -1,
                       'ftsprobe process': -1,'sweeper process': -1,'wal sender process': -1}
       for process in dict_process:
           pid = self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword=process)
           dict_process[process] = pid 
       self.kill_transc_backend()
       for process in dict_process:
           pid = self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword=process)
           delay = 1
           while dict_process.get(process) == pid and delay < 5:
                pid = self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword=process)
                sleep(1)
                delay = delay +1
           if delay == 5:
                tinctest.logger.error("Killing transaction backend process failed to force postmaster reset: %s"%process)
                raise Exception("Killing transaction backend process failed to force postmaster reset child process") 
 
   def kill_transc_backend(self):
       pid_list = []
       sql = "SELECT procpid FROM pg_stat_activity WHERE datname='{0}' AND current_query like 'INSERT INTO%'".format(self.pgdatabase)    
       tinctest.logger.info("running sql command to get transaction backend process: ---  %s"%sql)
       procid = PSQL.run_sql_command(sql, flags = '-q -t', dbname= self.pgdatabase)
#.........这里部分代码省略.........

 def __init__(self,methodName):
     self.gputil = GpUtility()
     self.stdby = StandbyVerify()
     super(GpstartTestCase,self).__init__(methodName)

class GpstartTestCase(MPPTestCase):
    '''
    testcase for gpstart

    gpstart may return status code 1 as well as 0 in the success case.  The
    difference is whether it produces WARNING or not, but here we don't care.
    '''
    origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')

    def __init__(self,methodName):
        self.gputil = GpUtility()
        self.stdby = StandbyVerify()
        super(GpstartTestCase,self).__init__(methodName)

    def setUp(self):
        self.gputil.check_and_start_gpdb()
        stdby_presence = self.gputil.check_standby_presence()
        # We should forcibly recreate standby, as it might has been promoted.
        if stdby_presence:
            self.gputil.remove_standby()
        self.gputil.install_standby()

    def tearDown(self):
        self.gputil.remove_standby()
        
    """
    Gpstart test cases in recovery mode
    """

    def test_gpstart_from_master(self):
        """
        tag
        """
        self.gputil.check_and_stop_gpdb()
        (rc, stdout) = self.gputil.run('gpstart -a ')
        self.assertIn(rc, (0, 1))
        self.assertTrue(self.gputil.gpstart_and_verify())
        sleep(2)
        self.assertTrue(self.stdby.check_gp_segment_config(),'standby master not cofigured')
        self.assertTrue(self.stdby.check_pg_stat_replication(),'standby not in replication status')
        self.assertTrue(self.stdby.check_standby_processes(), 'standby processes not running')
        (rc, output) = self.gputil.run(command = 'ps -ef|grep "wal sender "|grep -v grep')
        self.assertIsNotNone(output)

    def test_gpstart_master_only(self):
        """
        tag
        """
        self.gputil.check_and_stop_gpdb()
        (rc, stdout) = self.gputil.run('export GPSTART_INTERNAL_MASTER_ONLY=1; '
                                   'gpstart -a -m ')
        self.assertIn(rc, (0, 1))
        self.assertTrue(self.gputil.gpstart_and_verify())
        (rc,output) = self.gputil.run('PGDATABASE=template1 '
                                  "PGOPTIONS='-c gp_session_role=utility' "
                                  'psql')
        self.assertEqual(rc, 0)
        (rc, output) = self.gputil.run('psql template1')
        # should fail due to master only mode
        self.assertEqual(rc, 2)
        self.gputil.run('gpstop -a -m')
        self.gputil.run('gpstart -a')


    def test_gpstart_restricted_mode_master(self):
        """Test -R option with standby."""

        self.gputil.check_and_stop_gpdb()
        (rc, stdout) = self.gputil.run('gpstart -a -R')
        self.assertIn(rc, (0, 1))
        self.assertTrue(self.gputil.gpstart_and_verify())
        (rc,output) = self.gputil.run(command = 'psql template1')
        self.assertIn(rc, (0, 1))
        self.gputil.run('gpstop -ar')


    def test_gpstart_master_w_timeout(self):
        """Test -t option with standby."""

        self.gputil.check_and_stop_gpdb()
        (rc, output) = self.gputil.run('gpstart -a -t 30')
        self.assertIn(rc, (0, 1))
        self.assertTrue(self.gputil.gpstart_and_verify())
        self.gputil.run('gpstop -ar')

    def test_gpstart_no_standby(self):
        """Test -y with standby configured."""

        self.gputil.check_and_stop_gpdb()
        (rc, stdout) = self.gputil.run('gpstart -a -y')
        self.assertIn(rc, (0, 1))
        self.assertTrue(self.gputil.gpstart_and_verify())
        self.assertFalse(self.stdby.check_standby_processes(),
                         'gpstart without standby failed, standby was running')
        self.gputil.run('gpstop -ar')

    def test_gpstart_wo_standby(self):
        """Test -y without standby configured."""

        self.gputil.remove_standby()
#.........这里部分代码省略.........

 def __init__(self, methodName):
     self.gp = GpinitStandby()
     self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
     self.pgutil = GpUtility()
     super(WalRecTestCase,self).__init__(methodName)

 def __init__(self,methodName):
     self.pgutil = GpUtility()
     super(PromoteTestCase,self).__init__(methodName)