<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0304</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2011-10-21</periodOfReport>

    <notSubjectToSection16>1</notSubjectToSection16>

    <issuer>
        <issuerCik>0000810689</issuerCik>
        <issuerName>BANK OF GRANITE CORP</issuerName>
        <issuerTradingSymbol>GRAN</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001210334</rptOwnerCik>
            <rptOwnerName>STEPHENS DARRYL MARK</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>1410 4TH STREET DRIVE, NW</rptOwnerStreet1>
            <rptOwnerStreet2>UNIT 301</rptOwnerStreet2>
            <rptOwnerCity>HICKORY</rptOwnerCity>
            <rptOwnerState>NC</rptOwnerState>
            <rptOwnerZipCode>28601</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>0</isDirector>
            <isOfficer>1</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle>Senior Vice President &amp; CIO</officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2011-10-21</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>4337</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F1"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2011-10-21</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>1472.5065</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F2"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>Held in Bank of Granite Employees Profit Sharing Plan</value>
                    <footnoteId id="F3"/>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
    </nonDerivativeTable>

    <derivativeTable></derivativeTable>

    <footnotes>
        <footnote id="F1">Disposed of pursuant to that certain Agreement and Plan of Merger, dated as of April 26, 2011, as amended by Amendment No. 1, dated as of June 16, 2011, and Amendment No. 2, dated as of August 15, 2011, by and among FNB United Corp., Gamma Merger Corporation and the issuer (the &quot;Merger&quot;) in exchange for 14,638 shares of FNB United Corp. common stock having a market value of $0.38 per share on the effective date of the Merger.</footnote>
        <footnote id="F2">Disposed of pursuant to the Merger in exchange for 4,970 shares of FNB United Corp. common stock having a market value of $0.38 per share on the effective date of the Merger.</footnote>
        <footnote id="F3">Held by issuer's Employees' Profit Sharing Plan. Column 4 includes a change of 846.02 shares in the Employees' Profit Sharing Plan since the date of the last ownership report.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/Kathy K. Starnes, Attorney in Fact</signatureName>
        <signatureDate>2011-10-25</signatureDate>
    </ownerSignature>
</ownershipDocument>
